summaryrefslogtreecommitdiffstats
path: root/drivers/misc
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2019-05-07 22:39:22 +0200
committerLinus Torvalds <torvalds@linux-foundation.org>2019-05-07 22:39:22 +0200
commitf678d6da749983791850876e3421e7c48a0a7127 (patch)
tree553f818ef8e73bf9d6b1e53bdf623240c1279ffb /drivers/misc
parentMerge tag 'char-misc-5.2-rc1-part1' of git://git.kernel.org/pub/scm/linux/ker... (diff)
parentintel_th: msu: Add current window tracking (diff)
downloadlinux-f678d6da749983791850876e3421e7c48a0a7127.tar.xz
linux-f678d6da749983791850876e3421e7c48a0a7127.zip
Merge tag 'char-misc-5.2-rc1-part2' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc
Pull char/misc update part 2 from Greg KH: "Here is the "real" big set of char/misc driver patches for 5.2-rc1 Loads of different driver subsystem stuff in here, all over the places: - thunderbolt driver updates - habanalabs driver updates - nvmem driver updates - extcon driver updates - intel_th driver updates - mei driver updates - coresight driver updates - soundwire driver cleanups and updates - fastrpc driver updates - other minor driver updates - chardev minor fixups Feels like this tree is getting to be a dumping ground of "small driver subsystems" these days. Which is fine with me, if it makes things easier for those subsystem maintainers. All of these have been in linux-next for a while with no reported issues" * tag 'char-misc-5.2-rc1-part2' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc: (255 commits) intel_th: msu: Add current window tracking intel_th: msu: Add a sysfs attribute to trigger window switch intel_th: msu: Correct the block wrap detection intel_th: Add switch triggering support intel_th: gth: Factor out trace start/stop intel_th: msu: Factor out pipeline draining intel_th: msu: Switch over to scatterlist intel_th: msu: Replace open-coded list_{first,last,next}_entry variants intel_th: Only report useful IRQs to subdevices intel_th: msu: Start handling IRQs intel_th: pci: Use MSI interrupt signalling intel_th: Communicate IRQ via resource intel_th: Add "rtit" source device intel_th: Skip subdevices if their MMIO is missing intel_th: Rework resource passing between glue layers and core intel_th: SPDX-ify the documentation intel_th: msu: Fix single mode with IOMMU coresight: funnel: Support static funnel dt-bindings: arm: coresight: Unify funnel DT binding coresight: replicator: Add new device id for static replicator ...
Diffstat (limited to 'drivers/misc')
-rw-r--r--drivers/misc/Kconfig8
-rw-r--r--drivers/misc/Makefile1
-rw-r--r--drivers/misc/aspeed-p2a-ctrl.c444
-rw-r--r--drivers/misc/cardreader/rts5260.c4
-rw-r--r--drivers/misc/fastrpc.c235
-rw-r--r--drivers/misc/genwqe/card_debugfs.c4
-rw-r--r--drivers/misc/habanalabs/Makefile2
-rw-r--r--drivers/misc/habanalabs/command_buffer.c13
-rw-r--r--drivers/misc/habanalabs/command_submission.c22
-rw-r--r--drivers/misc/habanalabs/context.c4
-rw-r--r--drivers/misc/habanalabs/debugfs.c96
-rw-r--r--drivers/misc/habanalabs/device.c93
-rw-r--r--drivers/misc/habanalabs/firmware_if.c322
-rw-r--r--drivers/misc/habanalabs/goya/Makefile3
-rw-r--r--drivers/misc/habanalabs/goya/goya.c1209
-rw-r--r--drivers/misc/habanalabs/goya/goyaP.h81
-rw-r--r--drivers/misc/habanalabs/goya/goya_coresight.c628
-rw-r--r--drivers/misc/habanalabs/goya/goya_security.c15
-rw-r--r--drivers/misc/habanalabs/habanalabs.h220
-rw-r--r--drivers/misc/habanalabs/habanalabs_drv.c9
-rw-r--r--drivers/misc/habanalabs/habanalabs_ioctl.c139
-rw-r--r--drivers/misc/habanalabs/hw_queue.c46
-rw-r--r--drivers/misc/habanalabs/include/armcp_if.h2
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/cpu_ca53_cfg_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/cpu_ca53_cfg_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/cpu_if_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/cpu_pll_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_0_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_1_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_2_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_3_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_4_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_macro_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_macro_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_nrtr_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_nrtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_0_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_0_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_1_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_2_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_3_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_4_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h12
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h3
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/ic_pll_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mc_pll_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme1_rtr_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme1_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme2_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme3_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme4_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme5_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme6_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme_cmdq_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme_cmdq_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme_qm_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme_qm_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mme_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mmu_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/mmu_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/pci_nrtr_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/pci_nrtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/pcie_aux_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/pcie_wrap_regs.h306
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/psoc_emmc_pll_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/psoc_global_conf_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/psoc_global_conf_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/psoc_mme_pll_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/psoc_pci_pll_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/psoc_spi_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x0_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x1_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x2_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x3_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x4_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/stlb_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/stlb_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cfg_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cfg_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cmdq_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cmdq_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc0_eml_cfg_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc0_eml_cfg_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc0_nrtr_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc0_nrtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc0_qm_masks.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc0_qm_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc1_cfg_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc1_cmdq_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc1_qm_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc1_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc2_cfg_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc2_cmdq_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc2_qm_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc2_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc3_cfg_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc3_cmdq_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc3_qm_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc3_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc4_cfg_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc4_cmdq_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc4_qm_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc4_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc5_cfg_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc5_cmdq_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc5_qm_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc5_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc6_cfg_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc6_cmdq_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc6_qm_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc6_rtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc7_cfg_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc7_cmdq_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc7_nrtr_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc7_qm_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/asic_reg/tpc_pll_regs.h1
-rw-r--r--drivers/misc/habanalabs/include/goya/goya.h4
-rw-r--r--drivers/misc/habanalabs/include/goya/goya_async_events.h9
-rw-r--r--drivers/misc/habanalabs/include/goya/goya_coresight.h199
-rw-r--r--drivers/misc/habanalabs/include/goya/goya_fw_if.h2
-rw-r--r--drivers/misc/habanalabs/include/hl_boot_if.h3
-rw-r--r--drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h16
-rw-r--r--drivers/misc/habanalabs/include/hw_ip/pci/pci_general.h23
-rw-r--r--drivers/misc/habanalabs/irq.c14
-rw-r--r--drivers/misc/habanalabs/memory.c197
-rw-r--r--drivers/misc/habanalabs/mmu.c600
-rw-r--r--drivers/misc/habanalabs/pci.c408
-rw-r--r--drivers/misc/kgdbts.c4
-rw-r--r--drivers/misc/mei/Kconfig12
-rw-r--r--drivers/misc/mei/Makefile2
-rw-r--r--drivers/misc/mei/bus-fixup.c14
-rw-r--r--drivers/misc/mei/bus.c13
-rw-r--r--drivers/misc/mei/client.c16
-rw-r--r--drivers/misc/mei/client.h14
-rw-r--r--drivers/misc/mei/debugfs.c15
-rw-r--r--drivers/misc/mei/dma-ring.c2
-rw-r--r--drivers/misc/mei/hbm.c15
-rw-r--r--drivers/misc/mei/hbm.h14
-rw-r--r--drivers/misc/mei/hdcp/Kconfig13
-rw-r--r--drivers/misc/mei/hdcp/Makefile2
-rw-r--r--drivers/misc/mei/hdcp/mei_hdcp.c2
-rw-r--r--drivers/misc/mei/hdcp/mei_hdcp.h2
-rw-r--r--drivers/misc/mei/hw-me-regs.h68
-rw-r--r--drivers/misc/mei/hw-me.c14
-rw-r--r--drivers/misc/mei/hw-me.h16
-rw-r--r--drivers/misc/mei/hw-txe-regs.h63
-rw-r--r--drivers/misc/mei/hw-txe.c14
-rw-r--r--drivers/misc/mei/hw-txe.h14
-rw-r--r--drivers/misc/mei/hw.h14
-rw-r--r--drivers/misc/mei/init.c34
-rw-r--r--drivers/misc/mei/interrupt.c15
-rw-r--r--drivers/misc/mei/main.c80
-rw-r--r--drivers/misc/mei/mei-trace.c14
-rw-r--r--drivers/misc/mei/mei-trace.h14
-rw-r--r--drivers/misc/mei/mei_dev.h17
-rw-r--r--drivers/misc/mei/pci-me.c15
-rw-r--r--drivers/misc/mei/pci-txe.c14
-rw-r--r--drivers/misc/sgi-xp/xpc_uv.c1
159 files changed, 4086 insertions, 1948 deletions
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index 42ab8ec92a04..3209ee020b15 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -496,6 +496,14 @@ config VEXPRESS_SYSCFG
bus. System Configuration interface is one of the possible means
of generating transactions on this bus.
+config ASPEED_P2A_CTRL
+ depends on (ARCH_ASPEED || COMPILE_TEST) && REGMAP && MFD_SYSCON
+ tristate "Aspeed ast2400/2500 HOST P2A VGA MMIO to BMC bridge control"
+ help
+ Control Aspeed ast2400/2500 HOST P2A VGA MMIO to BMC mappings through
+ ioctl()s, the driver also provides an interface for userspace mappings to
+ a pre-defined region.
+
config ASPEED_LPC_CTRL
depends on (ARCH_ASPEED || COMPILE_TEST) && REGMAP && MFD_SYSCON
tristate "Aspeed ast2400/2500 HOST LPC to BMC bridge control"
diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
index d5b7d3404dc7..c36239573a5c 100644
--- a/drivers/misc/Makefile
+++ b/drivers/misc/Makefile
@@ -56,6 +56,7 @@ obj-$(CONFIG_VEXPRESS_SYSCFG) += vexpress-syscfg.o
obj-$(CONFIG_CXL_BASE) += cxl/
obj-$(CONFIG_ASPEED_LPC_CTRL) += aspeed-lpc-ctrl.o
obj-$(CONFIG_ASPEED_LPC_SNOOP) += aspeed-lpc-snoop.o
+obj-$(CONFIG_ASPEED_P2A_CTRL) += aspeed-p2a-ctrl.o
obj-$(CONFIG_PCI_ENDPOINT_TEST) += pci_endpoint_test.o
obj-$(CONFIG_OCXL) += ocxl/
obj-y += cardreader/
diff --git a/drivers/misc/aspeed-p2a-ctrl.c b/drivers/misc/aspeed-p2a-ctrl.c
new file mode 100644
index 000000000000..b60fbeaffcbd
--- /dev/null
+++ b/drivers/misc/aspeed-p2a-ctrl.c
@@ -0,0 +1,444 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2019 Google Inc
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * Provides a simple driver to control the ASPEED P2A interface which allows
+ * the host to read and write to various regions of the BMC's memory.
+ */
+
+#include <linux/fs.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/miscdevice.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+
+#include <linux/aspeed-p2a-ctrl.h>
+
+#define DEVICE_NAME "aspeed-p2a-ctrl"
+
+/* SCU2C is a Misc. Control Register. */
+#define SCU2C 0x2c
+/* SCU180 is the PCIe Configuration Setting Control Register. */
+#define SCU180 0x180
+/* Bit 1 controls the P2A bridge, while bit 0 controls the entire VGA device
+ * on the PCI bus.
+ */
+#define SCU180_ENP2A BIT(1)
+
+/* The ast2400/2500 both have six ranges. */
+#define P2A_REGION_COUNT 6
+
+struct region {
+ u64 min;
+ u64 max;
+ u32 bit;
+};
+
+struct aspeed_p2a_model_data {
+ /* min, max, bit */
+ struct region regions[P2A_REGION_COUNT];
+};
+
+struct aspeed_p2a_ctrl {
+ struct miscdevice miscdev;
+ struct regmap *regmap;
+
+ const struct aspeed_p2a_model_data *config;
+
+ /* Access to these needs to be locked, held via probe, mapping ioctl,
+ * and release, remove.
+ */
+ struct mutex tracking;
+ u32 readers;
+ u32 readerwriters[P2A_REGION_COUNT];
+
+ phys_addr_t mem_base;
+ resource_size_t mem_size;
+};
+
+struct aspeed_p2a_user {
+ struct file *file;
+ struct aspeed_p2a_ctrl *parent;
+
+ /* The entire memory space is opened for reading once the bridge is
+ * enabled, therefore this needs only to be tracked once per user.
+ * If any user has it open for read, the bridge must stay enabled.
+ */
+ u32 read;
+
+ /* Each entry of the array corresponds to a P2A Region. If the user
+ * opens for read or readwrite, the reference goes up here. On
+ * release, this array is walked and references adjusted accordingly.
+ */
+ u32 readwrite[P2A_REGION_COUNT];
+};
+
+static void aspeed_p2a_enable_bridge(struct aspeed_p2a_ctrl *p2a_ctrl)
+{
+ regmap_update_bits(p2a_ctrl->regmap,
+ SCU180, SCU180_ENP2A, SCU180_ENP2A);
+}
+
+static void aspeed_p2a_disable_bridge(struct aspeed_p2a_ctrl *p2a_ctrl)
+{
+ regmap_update_bits(p2a_ctrl->regmap, SCU180, SCU180_ENP2A, 0);
+}
+
+static int aspeed_p2a_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ unsigned long vsize;
+ pgprot_t prot;
+ struct aspeed_p2a_user *priv = file->private_data;
+ struct aspeed_p2a_ctrl *ctrl = priv->parent;
+
+ if (ctrl->mem_base == 0 && ctrl->mem_size == 0)
+ return -EINVAL;
+
+ vsize = vma->vm_end - vma->vm_start;
+ prot = vma->vm_page_prot;
+
+ if (vma->vm_pgoff + vsize > ctrl->mem_base + ctrl->mem_size)
+ return -EINVAL;
+
+ /* ast2400/2500 AHB accesses are not cache coherent */
+ prot = pgprot_noncached(prot);
+
+ if (remap_pfn_range(vma, vma->vm_start,
+ (ctrl->mem_base >> PAGE_SHIFT) + vma->vm_pgoff,
+ vsize, prot))
+ return -EAGAIN;
+
+ return 0;
+}
+
+static bool aspeed_p2a_region_acquire(struct aspeed_p2a_user *priv,
+ struct aspeed_p2a_ctrl *ctrl,
+ struct aspeed_p2a_ctrl_mapping *map)
+{
+ int i;
+ u64 base, end;
+ bool matched = false;
+
+ base = map->addr;
+ end = map->addr + (map->length - 1);
+
+ /* If the value is a legal u32, it will find a match. */
+ for (i = 0; i < P2A_REGION_COUNT; i++) {
+ const struct region *curr = &ctrl->config->regions[i];
+
+ /* If the top of this region is lower than your base, skip it.
+ */
+ if (curr->max < base)
+ continue;
+
+ /* If the bottom of this region is higher than your end, bail.
+ */
+ if (curr->min > end)
+ break;
+
+ /* Lock this and update it, therefore it someone else is
+ * closing their file out, this'll preserve the increment.
+ */
+ mutex_lock(&ctrl->tracking);
+ ctrl->readerwriters[i] += 1;
+ mutex_unlock(&ctrl->tracking);
+
+ /* Track with the user, so when they close their file, we can
+ * decrement properly.
+ */
+ priv->readwrite[i] += 1;
+
+ /* Enable the region as read-write. */
+ regmap_update_bits(ctrl->regmap, SCU2C, curr->bit, 0);
+ matched = true;
+ }
+
+ return matched;
+}
+
+static long aspeed_p2a_ioctl(struct file *file, unsigned int cmd,
+ unsigned long data)
+{
+ struct aspeed_p2a_user *priv = file->private_data;
+ struct aspeed_p2a_ctrl *ctrl = priv->parent;
+ void __user *arg = (void __user *)data;
+ struct aspeed_p2a_ctrl_mapping map;
+
+ if (copy_from_user(&map, arg, sizeof(map)))
+ return -EFAULT;
+
+ switch (cmd) {
+ case ASPEED_P2A_CTRL_IOCTL_SET_WINDOW:
+ /* If they want a region to be read-only, since the entire
+ * region is read-only once enabled, we just need to track this
+ * user wants to read from the bridge, and if it's not enabled.
+ * Enable it.
+ */
+ if (map.flags == ASPEED_P2A_CTRL_READ_ONLY) {
+ mutex_lock(&ctrl->tracking);
+ ctrl->readers += 1;
+ mutex_unlock(&ctrl->tracking);
+
+ /* Track with the user, so when they close their file,
+ * we can decrement properly.
+ */
+ priv->read += 1;
+ } else if (map.flags == ASPEED_P2A_CTRL_READWRITE) {
+ /* If we don't acquire any region return error. */
+ if (!aspeed_p2a_region_acquire(priv, ctrl, &map)) {
+ return -EINVAL;
+ }
+ } else {
+ /* Invalid map flags. */
+ return -EINVAL;
+ }
+
+ aspeed_p2a_enable_bridge(ctrl);
+ return 0;
+ case ASPEED_P2A_CTRL_IOCTL_GET_MEMORY_CONFIG:
+ /* This is a request for the memory-region and corresponding
+ * length that is used by the driver for mmap.
+ */
+
+ map.flags = 0;
+ map.addr = ctrl->mem_base;
+ map.length = ctrl->mem_size;
+
+ return copy_to_user(arg, &map, sizeof(map)) ? -EFAULT : 0;
+ }
+
+ return -EINVAL;
+}
+
+
+/*
+ * When a user opens this file, we create a structure to track their mappings.
+ *
+ * A user can map a region as read-only (bridge enabled), or read-write (bit
+ * flipped, and bridge enabled). Either way, this tracking is used, s.t. when
+ * they release the device references are handled.
+ *
+ * The bridge is not enabled until a user calls an ioctl to map a region,
+ * simply opening the device does not enable it.
+ */
+static int aspeed_p2a_open(struct inode *inode, struct file *file)
+{
+ struct aspeed_p2a_user *priv;
+
+ priv = kmalloc(sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->file = file;
+ priv->read = 0;
+ memset(priv->readwrite, 0, sizeof(priv->readwrite));
+
+ /* The file's private_data is initialized to the p2a_ctrl. */
+ priv->parent = file->private_data;
+
+ /* Set the file's private_data to the user's data. */
+ file->private_data = priv;
+
+ return 0;
+}
+
+/*
+ * This will close the users mappings. It will go through what they had opened
+ * for readwrite, and decrement those counts. If at the end, this is the last
+ * user, it'll close the bridge.
+ */
+static int aspeed_p2a_release(struct inode *inode, struct file *file)
+{
+ int i;
+ u32 bits = 0;
+ bool open_regions = false;
+ struct aspeed_p2a_user *priv = file->private_data;
+
+ /* Lock others from changing these values until everything is updated
+ * in one pass.
+ */
+ mutex_lock(&priv->parent->tracking);
+
+ priv->parent->readers -= priv->read;
+
+ for (i = 0; i < P2A_REGION_COUNT; i++) {
+ priv->parent->readerwriters[i] -= priv->readwrite[i];
+
+ if (priv->parent->readerwriters[i] > 0)
+ open_regions = true;
+ else
+ bits |= priv->parent->config->regions[i].bit;
+ }
+
+ /* Setting a bit to 1 disables the region, so let's just OR with the
+ * above to disable any.
+ */
+
+ /* Note, if another user is trying to ioctl, they can't grab tracking,
+ * and therefore can't grab either register mutex.
+ * If another user is trying to close, they can't grab tracking either.
+ */
+ regmap_update_bits(priv->parent->regmap, SCU2C, bits, bits);
+
+ /* If parent->readers is zero and open windows is 0, disable the
+ * bridge.
+ */
+ if (!open_regions && priv->parent->readers == 0)
+ aspeed_p2a_disable_bridge(priv->parent);
+
+ mutex_unlock(&priv->parent->tracking);
+
+ kfree(priv);
+
+ return 0;
+}
+
+static const struct file_operations aspeed_p2a_ctrl_fops = {
+ .owner = THIS_MODULE,
+ .mmap = aspeed_p2a_mmap,
+ .unlocked_ioctl = aspeed_p2a_ioctl,
+ .open = aspeed_p2a_open,
+ .release = aspeed_p2a_release,
+};
+
+/* The regions are controlled by SCU2C */
+static void aspeed_p2a_disable_all(struct aspeed_p2a_ctrl *p2a_ctrl)
+{
+ int i;
+ u32 value = 0;
+
+ for (i = 0; i < P2A_REGION_COUNT; i++)
+ value |= p2a_ctrl->config->regions[i].bit;
+
+ regmap_update_bits(p2a_ctrl->regmap, SCU2C, value, value);
+
+ /* Disable the bridge. */
+ aspeed_p2a_disable_bridge(p2a_ctrl);
+}
+
+static int aspeed_p2a_ctrl_probe(struct platform_device *pdev)
+{
+ struct aspeed_p2a_ctrl *misc_ctrl;
+ struct device *dev;
+ struct resource resm;
+ struct device_node *node;
+ int rc = 0;
+
+ dev = &pdev->dev;
+
+ misc_ctrl = devm_kzalloc(dev, sizeof(*misc_ctrl), GFP_KERNEL);
+ if (!misc_ctrl)
+ return -ENOMEM;
+
+ mutex_init(&misc_ctrl->tracking);
+
+ /* optional. */
+ node = of_parse_phandle(dev->of_node, "memory-region", 0);
+ if (node) {
+ rc = of_address_to_resource(node, 0, &resm);
+ of_node_put(node);
+ if (rc) {
+ dev_err(dev, "Couldn't address to resource for reserved memory\n");
+ return -ENODEV;
+ }
+
+ misc_ctrl->mem_size = resource_size(&resm);
+ misc_ctrl->mem_base = resm.start;
+ }
+
+ misc_ctrl->regmap = syscon_node_to_regmap(pdev->dev.parent->of_node);
+ if (IS_ERR(misc_ctrl->regmap)) {
+ dev_err(dev, "Couldn't get regmap\n");
+ return -ENODEV;
+ }
+
+ misc_ctrl->config = of_device_get_match_data(dev);
+
+ dev_set_drvdata(&pdev->dev, misc_ctrl);
+
+ aspeed_p2a_disable_all(misc_ctrl);
+
+ misc_ctrl->miscdev.minor = MISC_DYNAMIC_MINOR;
+ misc_ctrl->miscdev.name = DEVICE_NAME;
+ misc_ctrl->miscdev.fops = &aspeed_p2a_ctrl_fops;
+ misc_ctrl->miscdev.parent = dev;
+
+ rc = misc_register(&misc_ctrl->miscdev);
+ if (rc)
+ dev_err(dev, "Unable to register device\n");
+
+ return rc;
+}
+
+static int aspeed_p2a_ctrl_remove(struct platform_device *pdev)
+{
+ struct aspeed_p2a_ctrl *p2a_ctrl = dev_get_drvdata(&pdev->dev);
+
+ misc_deregister(&p2a_ctrl->miscdev);
+
+ return 0;
+}
+
+#define SCU2C_DRAM BIT(25)
+#define SCU2C_SPI BIT(24)
+#define SCU2C_SOC BIT(23)
+#define SCU2C_FLASH BIT(22)
+
+static const struct aspeed_p2a_model_data ast2400_model_data = {
+ .regions = {
+ {0x00000000, 0x17FFFFFF, SCU2C_FLASH},
+ {0x18000000, 0x1FFFFFFF, SCU2C_SOC},
+ {0x20000000, 0x2FFFFFFF, SCU2C_FLASH},
+ {0x30000000, 0x3FFFFFFF, SCU2C_SPI},
+ {0x40000000, 0x5FFFFFFF, SCU2C_DRAM},
+ {0x60000000, 0xFFFFFFFF, SCU2C_SOC},
+ }
+};
+
+static const struct aspeed_p2a_model_data ast2500_model_data = {
+ .regions = {
+ {0x00000000, 0x0FFFFFFF, SCU2C_FLASH},
+ {0x10000000, 0x1FFFFFFF, SCU2C_SOC},
+ {0x20000000, 0x3FFFFFFF, SCU2C_FLASH},
+ {0x40000000, 0x5FFFFFFF, SCU2C_SOC},
+ {0x60000000, 0x7FFFFFFF, SCU2C_SPI},
+ {0x80000000, 0xFFFFFFFF, SCU2C_DRAM},
+ }
+};
+
+static const struct of_device_id aspeed_p2a_ctrl_match[] = {
+ { .compatible = "aspeed,ast2400-p2a-ctrl",
+ .data = &ast2400_model_data },
+ { .compatible = "aspeed,ast2500-p2a-ctrl",
+ .data = &ast2500_model_data },
+ { },
+};
+
+static struct platform_driver aspeed_p2a_ctrl_driver = {
+ .driver = {
+ .name = DEVICE_NAME,
+ .of_match_table = aspeed_p2a_ctrl_match,
+ },
+ .probe = aspeed_p2a_ctrl_probe,
+ .remove = aspeed_p2a_ctrl_remove,
+};
+
+module_platform_driver(aspeed_p2a_ctrl_driver);
+
+MODULE_DEVICE_TABLE(of, aspeed_p2a_ctrl_match);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Patrick Venture <venture@google.com>");
+MODULE_DESCRIPTION("Control for aspeed 2400/2500 P2A VGA HOST to BMC mappings");
diff --git a/drivers/misc/cardreader/rts5260.c b/drivers/misc/cardreader/rts5260.c
index 52c95add56f0..4e285addbf2b 100644
--- a/drivers/misc/cardreader/rts5260.c
+++ b/drivers/misc/cardreader/rts5260.c
@@ -456,13 +456,13 @@ static void rts5260_pwr_saving_setting(struct rtsx_pcr *pcr)
pcr_dbg(pcr, "Set parameters for L1.2.");
rtsx_pci_write_register(pcr, PWR_GLOBAL_CTRL,
0xFF, PCIE_L1_2_EN);
- rtsx_pci_write_register(pcr, RTS5260_DVCC_CTRL,
+ rtsx_pci_write_register(pcr, RTS5260_DVCC_CTRL,
RTS5260_DVCC_OCP_EN |
RTS5260_DVCC_OCP_CL_EN,
RTS5260_DVCC_OCP_EN |
RTS5260_DVCC_OCP_CL_EN);
- rtsx_pci_write_register(pcr, PWR_FE_CTL,
+ rtsx_pci_write_register(pcr, PWR_FE_CTL,
0xFF, PCIE_L1_2_PD_FE_EN);
} else if (lss_l1_1) {
pcr_dbg(pcr, "Set parameters for L1.1.");
diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c
index 36d0d5c9cfba..98603e235cf0 100644
--- a/drivers/misc/fastrpc.c
+++ b/drivers/misc/fastrpc.c
@@ -12,6 +12,7 @@
#include <linux/module.h>
#include <linux/of_address.h>
#include <linux/of.h>
+#include <linux/sort.h>
#include <linux/of_platform.h>
#include <linux/rpmsg.h>
#include <linux/scatterlist.h>
@@ -31,7 +32,7 @@
#define FASTRPC_CTX_MAX (256)
#define FASTRPC_INIT_HANDLE 1
#define FASTRPC_CTXID_MASK (0xFF0)
-#define INIT_FILELEN_MAX (2 * 1024 * 1024)
+#define INIT_FILELEN_MAX (64 * 1024 * 1024)
#define INIT_MEMLEN_MAX (8 * 1024 * 1024)
#define FASTRPC_DEVICE_NAME "fastrpc"
@@ -104,6 +105,15 @@ struct fastrpc_invoke_rsp {
int retval; /* invoke return value */
};
+struct fastrpc_buf_overlap {
+ u64 start;
+ u64 end;
+ int raix;
+ u64 mstart;
+ u64 mend;
+ u64 offset;
+};
+
struct fastrpc_buf {
struct fastrpc_user *fl;
struct dma_buf *dmabuf;
@@ -149,12 +159,14 @@ struct fastrpc_invoke_ctx {
struct kref refcount;
struct list_head node; /* list of ctxs */
struct completion work;
+ struct work_struct put_work;
struct fastrpc_msg msg;
struct fastrpc_user *fl;
struct fastrpc_remote_arg *rpra;
struct fastrpc_map **maps;
struct fastrpc_buf *buf;
struct fastrpc_invoke_args *args;
+ struct fastrpc_buf_overlap *olaps;
struct fastrpc_channel_ctx *cctx;
};
@@ -282,6 +294,7 @@ static void fastrpc_context_free(struct kref *ref)
{
struct fastrpc_invoke_ctx *ctx;
struct fastrpc_channel_ctx *cctx;
+ unsigned long flags;
int i;
ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
@@ -293,11 +306,12 @@ static void fastrpc_context_free(struct kref *ref)
if (ctx->buf)
fastrpc_buf_free(ctx->buf);
- spin_lock(&cctx->lock);
+ spin_lock_irqsave(&cctx->lock, flags);
idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);
- spin_unlock(&cctx->lock);
+ spin_unlock_irqrestore(&cctx->lock, flags);
kfree(ctx->maps);
+ kfree(ctx->olaps);
kfree(ctx);
}
@@ -311,12 +325,70 @@ static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
kref_put(&ctx->refcount, fastrpc_context_free);
}
+static void fastrpc_context_put_wq(struct work_struct *work)
+{
+ struct fastrpc_invoke_ctx *ctx =
+ container_of(work, struct fastrpc_invoke_ctx, put_work);
+
+ fastrpc_context_put(ctx);
+}
+
+#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
+static int olaps_cmp(const void *a, const void *b)
+{
+ struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
+ struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
+ /* sort with lowest starting buffer first */
+ int st = CMP(pa->start, pb->start);
+ /* sort with highest ending buffer first */
+ int ed = CMP(pb->end, pa->end);
+
+ return st == 0 ? ed : st;
+}
+
+static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
+{
+ u64 max_end = 0;
+ int i;
+
+ for (i = 0; i < ctx->nbufs; ++i) {
+ ctx->olaps[i].start = ctx->args[i].ptr;
+ ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
+ ctx->olaps[i].raix = i;
+ }
+
+ sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
+
+ for (i = 0; i < ctx->nbufs; ++i) {
+ /* Falling inside previous range */
+ if (ctx->olaps[i].start < max_end) {
+ ctx->olaps[i].mstart = max_end;
+ ctx->olaps[i].mend = ctx->olaps[i].end;
+ ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
+
+ if (ctx->olaps[i].end > max_end) {
+ max_end = ctx->olaps[i].end;
+ } else {
+ ctx->olaps[i].mend = 0;
+ ctx->olaps[i].mstart = 0;
+ }
+
+ } else {
+ ctx->olaps[i].mend = ctx->olaps[i].end;
+ ctx->olaps[i].mstart = ctx->olaps[i].start;
+ ctx->olaps[i].offset = 0;
+ max_end = ctx->olaps[i].end;
+ }
+ }
+}
+
static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
struct fastrpc_user *user, u32 kernel, u32 sc,
struct fastrpc_invoke_args *args)
{
struct fastrpc_channel_ctx *cctx = user->cctx;
struct fastrpc_invoke_ctx *ctx = NULL;
+ unsigned long flags;
int ret;
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
@@ -336,7 +408,15 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
kfree(ctx);
return ERR_PTR(-ENOMEM);
}
+ ctx->olaps = kcalloc(ctx->nscalars,
+ sizeof(*ctx->olaps), GFP_KERNEL);
+ if (!ctx->olaps) {
+ kfree(ctx->maps);
+ kfree(ctx);
+ return ERR_PTR(-ENOMEM);
+ }
ctx->args = args;
+ fastrpc_get_buff_overlaps(ctx);
}
ctx->sc = sc;
@@ -345,20 +425,21 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
ctx->tgid = user->tgid;
ctx->cctx = cctx;
init_completion(&ctx->work);
+ INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
spin_lock(&user->lock);
list_add_tail(&ctx->node, &user->pending);
spin_unlock(&user->lock);
- spin_lock(&cctx->lock);
+ spin_lock_irqsave(&cctx->lock, flags);
ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,
FASTRPC_CTX_MAX, GFP_ATOMIC);
if (ret < 0) {
- spin_unlock(&cctx->lock);
+ spin_unlock_irqrestore(&cctx->lock, flags);
goto err_idr;
}
ctx->ctxid = ret << 4;
- spin_unlock(&cctx->lock);
+ spin_unlock_irqrestore(&cctx->lock, flags);
kref_init(&ctx->refcount);
@@ -368,6 +449,7 @@ err_idr:
list_del(&ctx->node);
spin_unlock(&user->lock);
kfree(ctx->maps);
+ kfree(ctx->olaps);
kfree(ctx);
return ERR_PTR(ret);
@@ -586,8 +668,11 @@ static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
size = ALIGN(metalen, FASTRPC_ALIGN);
for (i = 0; i < ctx->nscalars; i++) {
if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
- size = ALIGN(size, FASTRPC_ALIGN);
- size += ctx->args[i].length;
+
+ if (ctx->olaps[i].offset == 0)
+ size = ALIGN(size, FASTRPC_ALIGN);
+
+ size += (ctx->olaps[i].mend - ctx->olaps[i].mstart);
}
}
@@ -625,12 +710,12 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
struct fastrpc_remote_arg *rpra;
struct fastrpc_invoke_buf *list;
struct fastrpc_phy_page *pages;
- int inbufs, i, err = 0;
- u64 rlen, pkt_size;
+ int inbufs, i, oix, err = 0;
+ u64 len, rlen, pkt_size;
+ u64 pg_start, pg_end;
uintptr_t args;
int metalen;
-
inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
metalen = fastrpc_get_meta_size(ctx);
pkt_size = fastrpc_get_payload_size(ctx, metalen);
@@ -653,8 +738,11 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
rlen = pkt_size - metalen;
ctx->rpra = rpra;
- for (i = 0; i < ctx->nbufs; ++i) {
- u64 len = ctx->args[i].length;
+ for (oix = 0; oix < ctx->nbufs; ++oix) {
+ int mlen;
+
+ i = ctx->olaps[oix].raix;
+ len = ctx->args[i].length;
rpra[i].pv = 0;
rpra[i].len = len;
@@ -664,22 +752,45 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
if (!len)
continue;
- pages[i].size = roundup(len, PAGE_SIZE);
-
if (ctx->maps[i]) {
+ struct vm_area_struct *vma = NULL;
+
rpra[i].pv = (u64) ctx->args[i].ptr;
pages[i].addr = ctx->maps[i]->phys;
+
+ vma = find_vma(current->mm, ctx->args[i].ptr);
+ if (vma)
+ pages[i].addr += ctx->args[i].ptr -
+ vma->vm_start;
+
+ pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
+ pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
+ PAGE_SHIFT;
+ pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
+
} else {
- rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
- args = ALIGN(args, FASTRPC_ALIGN);
- if (rlen < len)
+
+ if (ctx->olaps[oix].offset == 0) {
+ rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
+ args = ALIGN(args, FASTRPC_ALIGN);
+ }
+
+ mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
+
+ if (rlen < mlen)
goto bail;
- rpra[i].pv = args;
- pages[i].addr = ctx->buf->phys + (pkt_size - rlen);
+ rpra[i].pv = args - ctx->olaps[oix].offset;
+ pages[i].addr = ctx->buf->phys -
+ ctx->olaps[oix].offset +
+ (pkt_size - rlen);
pages[i].addr = pages[i].addr & PAGE_MASK;
- args = args + len;
- rlen -= len;
+
+ pg_start = (args & PAGE_MASK) >> PAGE_SHIFT;
+ pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
+ pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
+ args = args + mlen;
+ rlen -= mlen;
}
if (i < inbufs && !ctx->maps[i]) {
@@ -782,6 +893,9 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
if (err)
goto bail;
}
+
+ /* make sure that all CPU memory writes are seen by DSP */
+ dma_wmb();
/* Send invoke buffer to remote dsp */
err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);
if (err)
@@ -798,6 +912,8 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
goto bail;
if (ctx->nscalars) {
+ /* make sure that all memory writes by DSP are seen by CPU */
+ dma_rmb();
/* populate all the output buffers with results */
err = fastrpc_put_args(ctx, kernel);
if (err)
@@ -843,12 +959,12 @@ static int fastrpc_init_create_process(struct fastrpc_user *fl,
if (copy_from_user(&init, argp, sizeof(init))) {
err = -EFAULT;
- goto bail;
+ goto err;
}
if (init.filelen > INIT_FILELEN_MAX) {
err = -EINVAL;
- goto bail;
+ goto err;
}
inbuf.pgid = fl->tgid;
@@ -862,17 +978,15 @@ static int fastrpc_init_create_process(struct fastrpc_user *fl,
if (init.filelen && init.filefd) {
err = fastrpc_map_create(fl, init.filefd, init.filelen, &map);
if (err)
- goto bail;
+ goto err;
}
memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1024 * 1024);
err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
&imem);
- if (err) {
- fastrpc_map_put(map);
- goto bail;
- }
+ if (err)
+ goto err_alloc;
fl->init_mem = imem;
args[0].ptr = (u64)(uintptr_t)&inbuf;
@@ -908,13 +1022,24 @@ static int fastrpc_init_create_process(struct fastrpc_user *fl,
err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
sc, args);
+ if (err)
+ goto err_invoke;
- if (err) {
+ kfree(args);
+
+ return 0;
+
+err_invoke:
+ fl->init_mem = NULL;
+ fastrpc_buf_free(imem);
+err_alloc:
+ if (map) {
+ spin_lock(&fl->lock);
+ list_del(&map->node);
+ spin_unlock(&fl->lock);
fastrpc_map_put(map);
- fastrpc_buf_free(imem);
}
-
-bail:
+err:
kfree(args);
return err;
@@ -924,9 +1049,10 @@ static struct fastrpc_session_ctx *fastrpc_session_alloc(
struct fastrpc_channel_ctx *cctx)
{
struct fastrpc_session_ctx *session = NULL;
+ unsigned long flags;
int i;
- spin_lock(&cctx->lock);
+ spin_lock_irqsave(&cctx->lock, flags);
for (i = 0; i < cctx->sesscount; i++) {
if (!cctx->session[i].used && cctx->session[i].valid) {
cctx->session[i].used = true;
@@ -934,7 +1060,7 @@ static struct fastrpc_session_ctx *fastrpc_session_alloc(
break;
}
}
- spin_unlock(&cctx->lock);
+ spin_unlock_irqrestore(&cctx->lock, flags);
return session;
}
@@ -942,9 +1068,11 @@ static struct fastrpc_session_ctx *fastrpc_session_alloc(
static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
struct fastrpc_session_ctx *session)
{
- spin_lock(&cctx->lock);
+ unsigned long flags;
+
+ spin_lock_irqsave(&cctx->lock, flags);
session->used = false;
- spin_unlock(&cctx->lock);
+ spin_unlock_irqrestore(&cctx->lock, flags);
}
static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
@@ -970,12 +1098,13 @@ static int fastrpc_device_release(struct inode *inode, struct file *file)
struct fastrpc_channel_ctx *cctx = fl->cctx;
struct fastrpc_invoke_ctx *ctx, *n;
struct fastrpc_map *map, *m;
+ unsigned long flags;
fastrpc_release_current_dsp_process(fl);
- spin_lock(&cctx->lock);
+ spin_lock_irqsave(&cctx->lock, flags);
list_del(&fl->user);
- spin_unlock(&cctx->lock);
+ spin_unlock_irqrestore(&cctx->lock, flags);
if (fl->init_mem)
fastrpc_buf_free(fl->init_mem);
@@ -1003,6 +1132,7 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp)
{
struct fastrpc_channel_ctx *cctx = miscdev_to_cctx(filp->private_data);
struct fastrpc_user *fl = NULL;
+ unsigned long flags;
fl = kzalloc(sizeof(*fl), GFP_KERNEL);
if (!fl)
@@ -1026,9 +1156,9 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp)
return -EBUSY;
}
- spin_lock(&cctx->lock);
+ spin_lock_irqsave(&cctx->lock, flags);
list_add_tail(&fl->user, &cctx->users);
- spin_unlock(&cctx->lock);
+ spin_unlock_irqrestore(&cctx->lock, flags);
return 0;
}
@@ -1184,6 +1314,7 @@ static int fastrpc_cb_probe(struct platform_device *pdev)
struct fastrpc_session_ctx *sess;
struct device *dev = &pdev->dev;
int i, sessions = 0;
+ unsigned long flags;
int rc;
cctx = dev_get_drvdata(dev->parent);
@@ -1192,7 +1323,7 @@ static int fastrpc_cb_probe(struct platform_device *pdev)
of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
- spin_lock(&cctx->lock);
+ spin_lock_irqsave(&cctx->lock, flags);
sess = &cctx->session[cctx->sesscount];
sess->used = false;
sess->valid = true;
@@ -1213,7 +1344,7 @@ static int fastrpc_cb_probe(struct platform_device *pdev)
}
}
cctx->sesscount++;
- spin_unlock(&cctx->lock);
+ spin_unlock_irqrestore(&cctx->lock, flags);
rc = dma_set_mask(dev, DMA_BIT_MASK(32));
if (rc) {
dev_err(dev, "32-bit DMA enable failed\n");
@@ -1227,16 +1358,17 @@ static int fastrpc_cb_remove(struct platform_device *pdev)
{
struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);
struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);
+ unsigned long flags;
int i;
- spin_lock(&cctx->lock);
+ spin_lock_irqsave(&cctx->lock, flags);
for (i = 1; i < FASTRPC_MAX_SESSIONS; i++) {
if (cctx->session[i].sid == sess->sid) {
cctx->session[i].valid = false;
cctx->sesscount--;
}
}
- spin_unlock(&cctx->lock);
+ spin_unlock_irqrestore(&cctx->lock, flags);
return 0;
}
@@ -1318,11 +1450,12 @@ static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
{
struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
struct fastrpc_user *user;
+ unsigned long flags;
- spin_lock(&cctx->lock);
+ spin_lock_irqsave(&cctx->lock, flags);
list_for_each_entry(user, &cctx->users, user)
fastrpc_notify_users(user);
- spin_unlock(&cctx->lock);
+ spin_unlock_irqrestore(&cctx->lock, flags);
misc_deregister(&cctx->miscdev);
of_platform_depopulate(&rpdev->dev);
@@ -1354,7 +1487,13 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
ctx->retval = rsp->retval;
complete(&ctx->work);
- fastrpc_context_put(ctx);
+
+ /*
+ * The DMA buffer associated with the context cannot be freed in
+ * interrupt context so schedule it through a worker thread to
+ * avoid a kernel BUG.
+ */
+ schedule_work(&ctx->put_work);
return 0;
}
diff --git a/drivers/misc/genwqe/card_debugfs.c b/drivers/misc/genwqe/card_debugfs.c
index 7c713e01d198..6f7e39f07811 100644
--- a/drivers/misc/genwqe/card_debugfs.c
+++ b/drivers/misc/genwqe/card_debugfs.c
@@ -227,7 +227,7 @@ static int ddcb_info_show(struct seq_file *s, void *unused)
seq_puts(s, "DDCB QUEUE:\n");
seq_printf(s, " ddcb_max: %d\n"
" ddcb_daddr: %016llx - %016llx\n"
- " ddcb_vaddr: %016llx\n"
+ " ddcb_vaddr: %p\n"
" ddcbs_in_flight: %u\n"
" ddcbs_max_in_flight: %u\n"
" ddcbs_completed: %u\n"
@@ -237,7 +237,7 @@ static int ddcb_info_show(struct seq_file *s, void *unused)
queue->ddcb_max, (long long)queue->ddcb_daddr,
(long long)queue->ddcb_daddr +
(queue->ddcb_max * DDCB_LENGTH),
- (long long)queue->ddcb_vaddr, queue->ddcbs_in_flight,
+ queue->ddcb_vaddr, queue->ddcbs_in_flight,
queue->ddcbs_max_in_flight, queue->ddcbs_completed,
queue->return_on_busy, queue->wait_on_busy,
cd->irqs_processed);
diff --git a/drivers/misc/habanalabs/Makefile b/drivers/misc/habanalabs/Makefile
index c6592db59b25..f8e85243d672 100644
--- a/drivers/misc/habanalabs/Makefile
+++ b/drivers/misc/habanalabs/Makefile
@@ -6,7 +6,7 @@ obj-m := habanalabs.o
habanalabs-y := habanalabs_drv.o device.o context.o asid.o habanalabs_ioctl.o \
command_buffer.o hw_queue.o irq.o sysfs.o hwmon.o memory.o \
- command_submission.o mmu.o
+ command_submission.o mmu.o firmware_if.o pci.o
habanalabs-$(CONFIG_DEBUG_FS) += debugfs.o
diff --git a/drivers/misc/habanalabs/command_buffer.c b/drivers/misc/habanalabs/command_buffer.c
index 85f75806a9a7..e495f44064fa 100644
--- a/drivers/misc/habanalabs/command_buffer.c
+++ b/drivers/misc/habanalabs/command_buffer.c
@@ -13,7 +13,7 @@
static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
{
- hdev->asic_funcs->dma_free_coherent(hdev, cb->size,
+ hdev->asic_funcs->asic_dma_free_coherent(hdev, cb->size,
(void *) (uintptr_t) cb->kernel_address,
cb->bus_address);
kfree(cb);
@@ -66,10 +66,10 @@ static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
return NULL;
if (ctx_id == HL_KERNEL_ASID_ID)
- p = hdev->asic_funcs->dma_alloc_coherent(hdev, cb_size,
+ p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
&cb->bus_address, GFP_ATOMIC);
else
- p = hdev->asic_funcs->dma_alloc_coherent(hdev, cb_size,
+ p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
&cb->bus_address,
GFP_USER | __GFP_ZERO);
if (!p) {
@@ -214,6 +214,13 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
u64 handle;
int rc;
+ if (hl_device_disabled_or_in_reset(hdev)) {
+ dev_warn_ratelimited(hdev->dev,
+ "Device is %s. Can't execute CB IOCTL\n",
+ atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
+ return -EBUSY;
+ }
+
switch (args->in.op) {
case HL_CB_OP_CREATE:
rc = hl_cb_create(hdev, &hpriv->cb_mgr, args->in.cb_size,
diff --git a/drivers/misc/habanalabs/command_submission.c b/drivers/misc/habanalabs/command_submission.c
index 19c84214a7ea..6fe785e26859 100644
--- a/drivers/misc/habanalabs/command_submission.c
+++ b/drivers/misc/habanalabs/command_submission.c
@@ -93,7 +93,6 @@ static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job)
parser.user_cb_size = job->user_cb_size;
parser.ext_queue = job->ext_queue;
job->patched_cb = NULL;
- parser.use_virt_addr = hdev->mmu_enable;
rc = hdev->asic_funcs->cs_parser(hdev, &parser);
if (job->ext_queue) {
@@ -261,7 +260,8 @@ static void cs_timedout(struct work_struct *work)
ctx_asid = cs->ctx->asid;
/* TODO: add information about last signaled seq and last emitted seq */
- dev_err(hdev->dev, "CS %d.%llu got stuck!\n", ctx_asid, cs->sequence);
+ dev_err(hdev->dev, "User %d command submission %llu got stuck!\n",
+ ctx_asid, cs->sequence);
cs_put(cs);
@@ -600,20 +600,20 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
void __user *chunks;
u32 num_chunks;
u64 cs_seq = ULONG_MAX;
- int rc, do_restore;
+ int rc, do_ctx_switch;
bool need_soft_reset = false;
if (hl_device_disabled_or_in_reset(hdev)) {
- dev_warn(hdev->dev,
+ dev_warn_ratelimited(hdev->dev,
"Device is %s. Can't submit new CS\n",
atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
rc = -EBUSY;
goto out;
}
- do_restore = atomic_cmpxchg(&ctx->thread_restore_token, 1, 0);
+ do_ctx_switch = atomic_cmpxchg(&ctx->thread_ctx_switch_token, 1, 0);
- if (do_restore || (args->in.cs_flags & HL_CS_FLAGS_FORCE_RESTORE)) {
+ if (do_ctx_switch || (args->in.cs_flags & HL_CS_FLAGS_FORCE_RESTORE)) {
long ret;
chunks = (void __user *)(uintptr_t)args->in.chunks_restore;
@@ -621,7 +621,7 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
mutex_lock(&hpriv->restore_phase_mutex);
- if (do_restore) {
+ if (do_ctx_switch) {
rc = hdev->asic_funcs->context_switch(hdev, ctx->asid);
if (rc) {
dev_err_ratelimited(hdev->dev,
@@ -677,18 +677,18 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
}
}
- ctx->thread_restore_wait_token = 1;
- } else if (!ctx->thread_restore_wait_token) {
+ ctx->thread_ctx_switch_wait_token = 1;
+ } else if (!ctx->thread_ctx_switch_wait_token) {
u32 tmp;
rc = hl_poll_timeout_memory(hdev,
- (u64) (uintptr_t) &ctx->thread_restore_wait_token,
+ (u64) (uintptr_t) &ctx->thread_ctx_switch_wait_token,
jiffies_to_usecs(hdev->timeout_jiffies),
&tmp);
if (rc || !tmp) {
dev_err(hdev->dev,
- "restore phase hasn't finished in time\n");
+ "context switch phase didn't finish in time\n");
rc = -ETIMEDOUT;
goto out;
}
diff --git a/drivers/misc/habanalabs/context.c b/drivers/misc/habanalabs/context.c
index 619ace1c4ef7..4804cdcf4c48 100644
--- a/drivers/misc/habanalabs/context.c
+++ b/drivers/misc/habanalabs/context.c
@@ -106,8 +106,8 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
ctx->cs_sequence = 1;
spin_lock_init(&ctx->cs_lock);
- atomic_set(&ctx->thread_restore_token, 1);
- ctx->thread_restore_wait_token = 0;
+ atomic_set(&ctx->thread_ctx_switch_token, 1);
+ ctx->thread_ctx_switch_wait_token = 0;
if (is_kernel_ctx) {
ctx->asid = HL_KERNEL_ASID_ID; /* KMD gets ASID 0 */
diff --git a/drivers/misc/habanalabs/debugfs.c b/drivers/misc/habanalabs/debugfs.c
index 974a87789bd8..a4447699ff4e 100644
--- a/drivers/misc/habanalabs/debugfs.c
+++ b/drivers/misc/habanalabs/debugfs.c
@@ -505,22 +505,97 @@ err:
return -EINVAL;
}
+static int device_va_to_pa(struct hl_device *hdev, u64 virt_addr,
+ u64 *phys_addr)
+{
+ struct hl_ctx *ctx = hdev->user_ctx;
+ u64 hop_addr, hop_pte_addr, hop_pte;
+ int rc = 0;
+
+ if (!ctx) {
+ dev_err(hdev->dev, "no ctx available\n");
+ return -EINVAL;
+ }
+
+ mutex_lock(&ctx->mmu_lock);
+
+ /* hop 0 */
+ hop_addr = get_hop0_addr(ctx);
+ hop_pte_addr = get_hop0_pte_addr(ctx, hop_addr, virt_addr);
+ hop_pte = hdev->asic_funcs->read_pte(hdev, hop_pte_addr);
+
+ /* hop 1 */
+ hop_addr = get_next_hop_addr(hop_pte);
+ if (hop_addr == ULLONG_MAX)
+ goto not_mapped;
+ hop_pte_addr = get_hop1_pte_addr(ctx, hop_addr, virt_addr);
+ hop_pte = hdev->asic_funcs->read_pte(hdev, hop_pte_addr);
+
+ /* hop 2 */
+ hop_addr = get_next_hop_addr(hop_pte);
+ if (hop_addr == ULLONG_MAX)
+ goto not_mapped;
+ hop_pte_addr = get_hop2_pte_addr(ctx, hop_addr, virt_addr);
+ hop_pte = hdev->asic_funcs->read_pte(hdev, hop_pte_addr);
+
+ /* hop 3 */
+ hop_addr = get_next_hop_addr(hop_pte);
+ if (hop_addr == ULLONG_MAX)
+ goto not_mapped;
+ hop_pte_addr = get_hop3_pte_addr(ctx, hop_addr, virt_addr);
+ hop_pte = hdev->asic_funcs->read_pte(hdev, hop_pte_addr);
+
+ if (!(hop_pte & LAST_MASK)) {
+ /* hop 4 */
+ hop_addr = get_next_hop_addr(hop_pte);
+ if (hop_addr == ULLONG_MAX)
+ goto not_mapped;
+ hop_pte_addr = get_hop4_pte_addr(ctx, hop_addr, virt_addr);
+ hop_pte = hdev->asic_funcs->read_pte(hdev, hop_pte_addr);
+ }
+
+ if (!(hop_pte & PAGE_PRESENT_MASK))
+ goto not_mapped;
+
+ *phys_addr = (hop_pte & PTE_PHYS_ADDR_MASK) | (virt_addr & OFFSET_MASK);
+
+ goto out;
+
+not_mapped:
+ dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n",
+ virt_addr);
+ rc = -EINVAL;
+out:
+ mutex_unlock(&ctx->mmu_lock);
+ return rc;
+}
+
static ssize_t hl_data_read32(struct file *f, char __user *buf,
size_t count, loff_t *ppos)
{
struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
struct hl_device *hdev = entry->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
char tmp_buf[32];
+ u64 addr = entry->addr;
u32 val;
ssize_t rc;
if (*ppos)
return 0;
- rc = hdev->asic_funcs->debugfs_read32(hdev, entry->addr, &val);
+ if (addr >= prop->va_space_dram_start_address &&
+ addr < prop->va_space_dram_end_address &&
+ hdev->mmu_enable &&
+ hdev->dram_supports_virtual_memory) {
+ rc = device_va_to_pa(hdev, entry->addr, &addr);
+ if (rc)
+ return rc;
+ }
+
+ rc = hdev->asic_funcs->debugfs_read32(hdev, addr, &val);
if (rc) {
- dev_err(hdev->dev, "Failed to read from 0x%010llx\n",
- entry->addr);
+ dev_err(hdev->dev, "Failed to read from 0x%010llx\n", addr);
return rc;
}
@@ -536,6 +611,8 @@ static ssize_t hl_data_write32(struct file *f, const char __user *buf,
{
struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
struct hl_device *hdev = entry->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u64 addr = entry->addr;
u32 value;
ssize_t rc;
@@ -543,10 +620,19 @@ static ssize_t hl_data_write32(struct file *f, const char __user *buf,
if (rc)
return rc;
- rc = hdev->asic_funcs->debugfs_write32(hdev, entry->addr, value);
+ if (addr >= prop->va_space_dram_start_address &&
+ addr < prop->va_space_dram_end_address &&
+ hdev->mmu_enable &&
+ hdev->dram_supports_virtual_memory) {
+ rc = device_va_to_pa(hdev, entry->addr, &addr);
+ if (rc)
+ return rc;
+ }
+
+ rc = hdev->asic_funcs->debugfs_write32(hdev, addr, value);
if (rc) {
dev_err(hdev->dev, "Failed to write 0x%08x to 0x%010llx\n",
- value, entry->addr);
+ value, addr);
return rc;
}
diff --git a/drivers/misc/habanalabs/device.c b/drivers/misc/habanalabs/device.c
index 77d51be66c7e..91a9e47a3482 100644
--- a/drivers/misc/habanalabs/device.c
+++ b/drivers/misc/habanalabs/device.c
@@ -5,11 +5,14 @@
* All Rights Reserved.
*/
+#define pr_fmt(fmt) "habanalabs: " fmt
+
#include "habanalabs.h"
#include <linux/pci.h>
#include <linux/sched/signal.h>
#include <linux/hwmon.h>
+#include <uapi/misc/habanalabs.h>
#define HL_PLDM_PENDING_RESET_PER_SEC (HL_PENDING_RESET_PER_SEC * 10)
@@ -21,6 +24,20 @@ bool hl_device_disabled_or_in_reset(struct hl_device *hdev)
return false;
}
+enum hl_device_status hl_device_status(struct hl_device *hdev)
+{
+ enum hl_device_status status;
+
+ if (hdev->disabled)
+ status = HL_DEVICE_STATUS_MALFUNCTION;
+ else if (atomic_read(&hdev->in_reset))
+ status = HL_DEVICE_STATUS_IN_RESET;
+ else
+ status = HL_DEVICE_STATUS_OPERATIONAL;
+
+ return status;
+};
+
static void hpriv_release(struct kref *ref)
{
struct hl_fpriv *hpriv;
@@ -498,11 +515,8 @@ disable_device:
return rc;
}
-static void hl_device_hard_reset_pending(struct work_struct *work)
+static void device_kill_open_processes(struct hl_device *hdev)
{
- struct hl_device_reset_work *device_reset_work =
- container_of(work, struct hl_device_reset_work, reset_work);
- struct hl_device *hdev = device_reset_work->hdev;
u16 pending_total, pending_cnt;
struct task_struct *task = NULL;
@@ -537,6 +551,12 @@ static void hl_device_hard_reset_pending(struct work_struct *work)
}
}
+ /* We killed the open users, but because the driver cleans up after the
+ * user contexts are closed (e.g. mmu mappings), we need to wait again
+ * to make sure the cleaning phase is finished before continuing with
+ * the reset
+ */
+
pending_cnt = pending_total;
while ((atomic_read(&hdev->fd_open_cnt)) && (pending_cnt)) {
@@ -552,6 +572,16 @@ static void hl_device_hard_reset_pending(struct work_struct *work)
mutex_unlock(&hdev->fd_open_cnt_lock);
+}
+
+static void device_hard_reset_pending(struct work_struct *work)
+{
+ struct hl_device_reset_work *device_reset_work =
+ container_of(work, struct hl_device_reset_work, reset_work);
+ struct hl_device *hdev = device_reset_work->hdev;
+
+ device_kill_open_processes(hdev);
+
hl_device_reset(hdev, true, true);
kfree(device_reset_work);
@@ -613,6 +643,8 @@ again:
if ((hard_reset) && (!from_hard_reset_thread)) {
struct hl_device_reset_work *device_reset_work;
+ hdev->hard_reset_pending = true;
+
if (!hdev->pdev) {
dev_err(hdev->dev,
"Reset action is NOT supported in simulator\n");
@@ -620,8 +652,6 @@ again:
goto out_err;
}
- hdev->hard_reset_pending = true;
-
device_reset_work = kzalloc(sizeof(*device_reset_work),
GFP_ATOMIC);
if (!device_reset_work) {
@@ -635,7 +665,7 @@ again:
* from a dedicated work
*/
INIT_WORK(&device_reset_work->reset_work,
- hl_device_hard_reset_pending);
+ device_hard_reset_pending);
device_reset_work->hdev = hdev;
schedule_work(&device_reset_work->reset_work);
@@ -663,17 +693,9 @@ again:
/* Go over all the queues, release all CS and their jobs */
hl_cs_rollback_all(hdev);
- if (hard_reset) {
- /* Release kernel context */
- if (hl_ctx_put(hdev->kernel_ctx) != 1) {
- dev_err(hdev->dev,
- "kernel ctx is alive during hard reset\n");
- rc = -EBUSY;
- goto out_err;
- }
-
+ /* Release kernel context */
+ if ((hard_reset) && (hl_ctx_put(hdev->kernel_ctx) == 1))
hdev->kernel_ctx = NULL;
- }
/* Reset the H/W. It will be in idle state after this returns */
hdev->asic_funcs->hw_fini(hdev, hard_reset);
@@ -688,16 +710,24 @@ again:
for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
hl_cq_reset(hdev, &hdev->completion_queue[i]);
- /* Make sure the setup phase for the user context will run again */
+ /* Make sure the context switch phase will run again */
if (hdev->user_ctx) {
- atomic_set(&hdev->user_ctx->thread_restore_token, 1);
- hdev->user_ctx->thread_restore_wait_token = 0;
+ atomic_set(&hdev->user_ctx->thread_ctx_switch_token, 1);
+ hdev->user_ctx->thread_ctx_switch_wait_token = 0;
}
/* Finished tear-down, starting to re-initialize */
if (hard_reset) {
hdev->device_cpu_disabled = false;
+ hdev->hard_reset_pending = false;
+
+ if (hdev->kernel_ctx) {
+ dev_crit(hdev->dev,
+ "kernel ctx was alive during hard reset, something is terribly wrong\n");
+ rc = -EBUSY;
+ goto out_err;
+ }
/* Allocate the kernel context */
hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
@@ -752,8 +782,6 @@ again:
}
hl_set_max_power(hdev, hdev->max_power);
-
- hdev->hard_reset_pending = false;
} else {
rc = hdev->asic_funcs->soft_reset_late_init(hdev);
if (rc) {
@@ -1030,11 +1058,22 @@ void hl_device_fini(struct hl_device *hdev)
WARN(1, "Failed to remove device because reset function did not finish\n");
return;
}
- };
+ }
/* Mark device as disabled */
hdev->disabled = true;
+ /*
+ * Flush anyone that is inside the critical section of enqueue
+ * jobs to the H/W
+ */
+ hdev->asic_funcs->hw_queues_lock(hdev);
+ hdev->asic_funcs->hw_queues_unlock(hdev);
+
+ hdev->hard_reset_pending = true;
+
+ device_kill_open_processes(hdev);
+
hl_hwmon_fini(hdev);
device_late_fini(hdev);
@@ -1108,7 +1147,13 @@ int hl_poll_timeout_memory(struct hl_device *hdev, u64 addr,
* either by the direct access of the device or by another core
*/
u32 *paddr = (u32 *) (uintptr_t) addr;
- ktime_t timeout = ktime_add_us(ktime_get(), timeout_us);
+ ktime_t timeout;
+
+ /* timeout should be longer when working with simulator */
+ if (!hdev->pdev)
+ timeout_us *= 10;
+
+ timeout = ktime_add_us(ktime_get(), timeout_us);
might_sleep();
diff --git a/drivers/misc/habanalabs/firmware_if.c b/drivers/misc/habanalabs/firmware_if.c
new file mode 100644
index 000000000000..eda5d7fcb79f
--- /dev/null
+++ b/drivers/misc/habanalabs/firmware_if.c
@@ -0,0 +1,322 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2019 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#include "habanalabs.h"
+
+#include <linux/firmware.h>
+#include <linux/genalloc.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+
+/**
+ * hl_fw_push_fw_to_device() - Push FW code to device.
+ * @hdev: pointer to hl_device structure.
+ *
+ * Copy fw code from firmware file to device memory.
+ *
+ * Return: 0 on success, non-zero for failure.
+ */
+int hl_fw_push_fw_to_device(struct hl_device *hdev, const char *fw_name,
+ void __iomem *dst)
+{
+ const struct firmware *fw;
+ const u64 *fw_data;
+ size_t fw_size, i;
+ int rc;
+
+ rc = request_firmware(&fw, fw_name, hdev->dev);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to request %s\n", fw_name);
+ goto out;
+ }
+
+ fw_size = fw->size;
+ if ((fw_size % 4) != 0) {
+ dev_err(hdev->dev, "illegal %s firmware size %zu\n",
+ fw_name, fw_size);
+ rc = -EINVAL;
+ goto out;
+ }
+
+ dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
+
+ fw_data = (const u64 *) fw->data;
+
+ if ((fw->size % 8) != 0)
+ fw_size -= 8;
+
+ for (i = 0 ; i < fw_size ; i += 8, fw_data++, dst += 8) {
+ if (!(i & (0x80000 - 1))) {
+ dev_dbg(hdev->dev,
+ "copied so far %zu out of %zu for %s firmware",
+ i, fw_size, fw_name);
+ usleep_range(20, 100);
+ }
+
+ writeq(*fw_data, dst);
+ }
+
+ if ((fw->size % 8) != 0)
+ writel(*(const u32 *) fw_data, dst);
+
+out:
+ release_firmware(fw);
+ return rc;
+}
+
+int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
+{
+ struct armcp_packet pkt = {};
+
+ pkt.ctl = cpu_to_le32(opcode << ARMCP_PKT_CTL_OPCODE_SHIFT);
+
+ return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
+ sizeof(pkt), HL_DEVICE_TIMEOUT_USEC, NULL);
+}
+
+int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
+ u16 len, u32 timeout, long *result)
+{
+ struct armcp_packet *pkt;
+ dma_addr_t pkt_dma_addr;
+ u32 tmp;
+ int rc = 0;
+
+ if (len > HL_CPU_CB_SIZE) {
+ dev_err(hdev->dev, "Invalid CPU message size of %d bytes\n",
+ len);
+ return -ENOMEM;
+ }
+
+ pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
+ &pkt_dma_addr);
+ if (!pkt) {
+ dev_err(hdev->dev,
+ "Failed to allocate DMA memory for packet to CPU\n");
+ return -ENOMEM;
+ }
+
+ memcpy(pkt, msg, len);
+
+ mutex_lock(&hdev->send_cpu_message_lock);
+
+ if (hdev->disabled)
+ goto out;
+
+ if (hdev->device_cpu_disabled) {
+ rc = -EIO;
+ goto out;
+ }
+
+ rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc);
+ goto out;
+ }
+
+ rc = hl_poll_timeout_memory(hdev, (u64) (uintptr_t) &pkt->fence,
+ timeout, &tmp);
+
+ hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
+
+ if (rc == -ETIMEDOUT) {
+ dev_err(hdev->dev, "Timeout while waiting for device CPU\n");
+ hdev->device_cpu_disabled = true;
+ goto out;
+ }
+
+ if (tmp == ARMCP_PACKET_FENCE_VAL) {
+ u32 ctl = le32_to_cpu(pkt->ctl);
+
+ rc = (ctl & ARMCP_PKT_CTL_RC_MASK) >> ARMCP_PKT_CTL_RC_SHIFT;
+ if (rc) {
+ dev_err(hdev->dev,
+ "F/W ERROR %d for CPU packet %d\n",
+ rc, (ctl & ARMCP_PKT_CTL_OPCODE_MASK)
+ >> ARMCP_PKT_CTL_OPCODE_SHIFT);
+ rc = -EINVAL;
+ } else if (result) {
+ *result = (long) le64_to_cpu(pkt->result);
+ }
+ } else {
+ dev_err(hdev->dev, "CPU packet wrong fence value\n");
+ rc = -EINVAL;
+ }
+
+out:
+ mutex_unlock(&hdev->send_cpu_message_lock);
+
+ hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
+
+ return rc;
+}
+
+int hl_fw_test_cpu_queue(struct hl_device *hdev)
+{
+ struct armcp_packet test_pkt = {};
+ long result;
+ int rc;
+
+ test_pkt.ctl = cpu_to_le32(ARMCP_PACKET_TEST <<
+ ARMCP_PKT_CTL_OPCODE_SHIFT);
+ test_pkt.value = cpu_to_le64(ARMCP_PACKET_FENCE_VAL);
+
+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
+ sizeof(test_pkt), HL_DEVICE_TIMEOUT_USEC, &result);
+
+ if (!rc) {
+ if (result == ARMCP_PACKET_FENCE_VAL)
+ dev_info(hdev->dev,
+ "queue test on CPU queue succeeded\n");
+ else
+ dev_err(hdev->dev,
+ "CPU queue test failed (0x%08lX)\n", result);
+ } else {
+ dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
+ }
+
+ return rc;
+}
+
+void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
+ dma_addr_t *dma_handle)
+{
+ u64 kernel_addr;
+
+ /* roundup to HL_CPU_PKT_SIZE */
+ size = (size + (HL_CPU_PKT_SIZE - 1)) & HL_CPU_PKT_MASK;
+
+ kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
+
+ *dma_handle = hdev->cpu_accessible_dma_address +
+ (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
+
+ return (void *) (uintptr_t) kernel_addr;
+}
+
+void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
+ void *vaddr)
+{
+ /* roundup to HL_CPU_PKT_SIZE */
+ size = (size + (HL_CPU_PKT_SIZE - 1)) & HL_CPU_PKT_MASK;
+
+ gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
+ size);
+}
+
+int hl_fw_send_heartbeat(struct hl_device *hdev)
+{
+ struct armcp_packet hb_pkt = {};
+ long result;
+ int rc;
+
+ hb_pkt.ctl = cpu_to_le32(ARMCP_PACKET_TEST <<
+ ARMCP_PKT_CTL_OPCODE_SHIFT);
+ hb_pkt.value = cpu_to_le64(ARMCP_PACKET_FENCE_VAL);
+
+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
+ sizeof(hb_pkt), HL_DEVICE_TIMEOUT_USEC, &result);
+
+ if ((rc) || (result != ARMCP_PACKET_FENCE_VAL))
+ rc = -EIO;
+
+ return rc;
+}
+
+int hl_fw_armcp_info_get(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct armcp_packet pkt = {};
+ void *armcp_info_cpu_addr;
+ dma_addr_t armcp_info_dma_addr;
+ long result;
+ int rc;
+
+ armcp_info_cpu_addr =
+ hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
+ sizeof(struct armcp_info),
+ &armcp_info_dma_addr);
+ if (!armcp_info_cpu_addr) {
+ dev_err(hdev->dev,
+ "Failed to allocate DMA memory for ArmCP info packet\n");
+ return -ENOMEM;
+ }
+
+ memset(armcp_info_cpu_addr, 0, sizeof(struct armcp_info));
+
+ pkt.ctl = cpu_to_le32(ARMCP_PACKET_INFO_GET <<
+ ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.addr = cpu_to_le64(armcp_info_dma_addr);
+ pkt.data_max_size = cpu_to_le32(sizeof(struct armcp_info));
+
+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+ HL_ARMCP_INFO_TIMEOUT_USEC, &result);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to send armcp info pkt, error %d\n", rc);
+ goto out;
+ }
+
+ memcpy(&prop->armcp_info, armcp_info_cpu_addr,
+ sizeof(prop->armcp_info));
+
+ rc = hl_build_hwmon_channel_info(hdev, prop->armcp_info.sensors);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to build hwmon channel info, error %d\n", rc);
+ rc = -EFAULT;
+ goto out;
+ }
+
+out:
+ hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
+ sizeof(struct armcp_info), armcp_info_cpu_addr);
+
+ return rc;
+}
+
+int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)
+{
+ struct armcp_packet pkt = {};
+ void *eeprom_info_cpu_addr;
+ dma_addr_t eeprom_info_dma_addr;
+ long result;
+ int rc;
+
+ eeprom_info_cpu_addr =
+ hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
+ max_size, &eeprom_info_dma_addr);
+ if (!eeprom_info_cpu_addr) {
+ dev_err(hdev->dev,
+ "Failed to allocate DMA memory for EEPROM info packet\n");
+ return -ENOMEM;
+ }
+
+ memset(eeprom_info_cpu_addr, 0, max_size);
+
+ pkt.ctl = cpu_to_le32(ARMCP_PACKET_EEPROM_DATA_GET <<
+ ARMCP_PKT_CTL_OPCODE_SHIFT);
+ pkt.addr = cpu_to_le64(eeprom_info_dma_addr);
+ pkt.data_max_size = cpu_to_le32(max_size);
+
+ rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+ HL_ARMCP_EEPROM_TIMEOUT_USEC, &result);
+
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to send armcp EEPROM pkt, error %d\n", rc);
+ goto out;
+ }
+
+ /* result contains the actual size */
+ memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
+
+out:
+ hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
+ eeprom_info_cpu_addr);
+
+ return rc;
+}
diff --git a/drivers/misc/habanalabs/goya/Makefile b/drivers/misc/habanalabs/goya/Makefile
index e458e5ba500b..131432f677e2 100644
--- a/drivers/misc/habanalabs/goya/Makefile
+++ b/drivers/misc/habanalabs/goya/Makefile
@@ -1,3 +1,4 @@
subdir-ccflags-y += -I$(src)
-HL_GOYA_FILES := goya/goya.o goya/goya_security.o goya/goya_hwmgr.o
+HL_GOYA_FILES := goya/goya.o goya/goya_security.o goya/goya_hwmgr.o \
+ goya/goya_coresight.o
diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c
index 3c509e19d69d..a582e29c1ee4 100644
--- a/drivers/misc/habanalabs/goya/goya.c
+++ b/drivers/misc/habanalabs/goya/goya.c
@@ -12,10 +12,8 @@
#include <linux/pci.h>
#include <linux/genalloc.h>
-#include <linux/firmware.h>
#include <linux/hwmon.h>
#include <linux/io-64-nonatomic-lo-hi.h>
-#include <linux/io-64-nonatomic-hi-lo.h>
/*
* GOYA security scheme:
@@ -71,7 +69,7 @@
*
*/
-#define GOYA_MMU_REGS_NUM 61
+#define GOYA_MMU_REGS_NUM 63
#define GOYA_DMA_POOL_BLK_SIZE 0x100 /* 256 bytes */
@@ -80,15 +78,12 @@
#define GOYA_RESET_WAIT_MSEC 1 /* 1ms */
#define GOYA_CPU_RESET_WAIT_MSEC 100 /* 100ms */
#define GOYA_PLDM_RESET_WAIT_MSEC 1000 /* 1s */
-#define GOYA_CPU_TIMEOUT_USEC 10000000 /* 10s */
#define GOYA_TEST_QUEUE_WAIT_USEC 100000 /* 100ms */
#define GOYA_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 100)
#define GOYA_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
#define GOYA_QMAN0_FENCE_VAL 0xD169B243
-#define GOYA_MAX_INITIATORS 20
-
#define GOYA_MAX_STRING_LEN 20
#define GOYA_CB_POOL_CB_CNT 512
@@ -173,12 +168,12 @@ static u64 goya_mmu_regs[GOYA_MMU_REGS_NUM] = {
mmMME_SBA_CONTROL_DATA,
mmMME_SBB_CONTROL_DATA,
mmMME_SBC_CONTROL_DATA,
- mmMME_WBC_CONTROL_DATA
+ mmMME_WBC_CONTROL_DATA,
+ mmPCIE_WRAP_PSOC_ARUSER,
+ mmPCIE_WRAP_PSOC_AWUSER
};
-#define GOYA_ASYC_EVENT_GROUP_NON_FATAL_SIZE 121
-
-static u32 goya_non_fatal_events[GOYA_ASYC_EVENT_GROUP_NON_FATAL_SIZE] = {
+static u32 goya_all_events[] = {
GOYA_ASYNC_EVENT_ID_PCIE_IF,
GOYA_ASYNC_EVENT_ID_TPC0_ECC,
GOYA_ASYNC_EVENT_ID_TPC1_ECC,
@@ -302,14 +297,7 @@ static u32 goya_non_fatal_events[GOYA_ASYC_EVENT_GROUP_NON_FATAL_SIZE] = {
GOYA_ASYNC_EVENT_ID_DMA_BM_CH4
};
-static int goya_armcp_info_get(struct hl_device *hdev);
-static void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
-static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
-static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
-static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
- u64 phys_addr);
-
-static void goya_get_fixed_properties(struct hl_device *hdev)
+void goya_get_fixed_properties(struct hl_device *hdev)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
int i;
@@ -357,7 +345,6 @@ static void goya_get_fixed_properties(struct hl_device *hdev)
prop->mmu_hop0_tables_total_size = HOP0_TABLES_TOTAL_SIZE;
prop->dram_page_size = PAGE_SIZE_2MB;
- prop->host_phys_base_address = HOST_PHYS_BASE;
prop->va_space_host_start_address = VA_HOST_SPACE_START;
prop->va_space_host_end_address = VA_HOST_SPACE_END;
prop->va_space_dram_start_address = VA_DDR_SPACE_START;
@@ -367,24 +354,13 @@ static void goya_get_fixed_properties(struct hl_device *hdev)
prop->cfg_size = CFG_SIZE;
prop->max_asid = MAX_ASID;
prop->num_of_events = GOYA_ASYNC_EVENT_ID_SIZE;
+ prop->high_pll = PLL_HIGH_DEFAULT;
prop->cb_pool_cb_cnt = GOYA_CB_POOL_CB_CNT;
prop->cb_pool_cb_size = GOYA_CB_POOL_CB_SIZE;
prop->max_power_default = MAX_POWER_DEFAULT;
prop->tpc_enabled_mask = TPC_ENABLED_MASK;
-
- prop->high_pll = PLL_HIGH_DEFAULT;
-}
-
-int goya_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
-{
- struct armcp_packet pkt;
-
- memset(&pkt, 0, sizeof(pkt));
-
- pkt.ctl = cpu_to_le32(opcode << ARMCP_PKT_CTL_OPCODE_SHIFT);
-
- return hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt,
- sizeof(pkt), HL_DEVICE_TIMEOUT_USEC, NULL);
+ prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
+ prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
}
/*
@@ -398,199 +374,40 @@ int goya_send_pci_access_msg(struct hl_device *hdev, u32 opcode)
*/
static int goya_pci_bars_map(struct hl_device *hdev)
{
- struct pci_dev *pdev = hdev->pdev;
+ static const char * const name[] = {"SRAM_CFG", "MSIX", "DDR"};
+ bool is_wc[3] = {false, false, true};
int rc;
- rc = pci_request_regions(pdev, HL_NAME);
- if (rc) {
- dev_err(hdev->dev, "Cannot obtain PCI resources\n");
+ rc = hl_pci_bars_map(hdev, name, is_wc);
+ if (rc)
return rc;
- }
-
- hdev->pcie_bar[SRAM_CFG_BAR_ID] =
- pci_ioremap_bar(pdev, SRAM_CFG_BAR_ID);
- if (!hdev->pcie_bar[SRAM_CFG_BAR_ID]) {
- dev_err(hdev->dev, "pci_ioremap_bar failed for CFG\n");
- rc = -ENODEV;
- goto err_release_regions;
- }
-
- hdev->pcie_bar[MSIX_BAR_ID] = pci_ioremap_bar(pdev, MSIX_BAR_ID);
- if (!hdev->pcie_bar[MSIX_BAR_ID]) {
- dev_err(hdev->dev, "pci_ioremap_bar failed for MSIX\n");
- rc = -ENODEV;
- goto err_unmap_sram_cfg;
- }
-
- hdev->pcie_bar[DDR_BAR_ID] = pci_ioremap_wc_bar(pdev, DDR_BAR_ID);
- if (!hdev->pcie_bar[DDR_BAR_ID]) {
- dev_err(hdev->dev, "pci_ioremap_bar failed for DDR\n");
- rc = -ENODEV;
- goto err_unmap_msix;
- }
hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] +
- (CFG_BASE - SRAM_BASE_ADDR);
-
- return 0;
-
-err_unmap_msix:
- iounmap(hdev->pcie_bar[MSIX_BAR_ID]);
-err_unmap_sram_cfg:
- iounmap(hdev->pcie_bar[SRAM_CFG_BAR_ID]);
-err_release_regions:
- pci_release_regions(pdev);
-
- return rc;
-}
-
-/*
- * goya_pci_bars_unmap - Unmap PCI BARS of Goya device
- *
- * @hdev: pointer to hl_device structure
- *
- * Release all PCI BARS and unmap their virtual addresses
- *
- */
-static void goya_pci_bars_unmap(struct hl_device *hdev)
-{
- struct pci_dev *pdev = hdev->pdev;
-
- iounmap(hdev->pcie_bar[DDR_BAR_ID]);
- iounmap(hdev->pcie_bar[MSIX_BAR_ID]);
- iounmap(hdev->pcie_bar[SRAM_CFG_BAR_ID]);
- pci_release_regions(pdev);
-}
-
-/*
- * goya_elbi_write - Write through the ELBI interface
- *
- * @hdev: pointer to hl_device structure
- *
- * return 0 on success, -1 on failure
- *
- */
-static int goya_elbi_write(struct hl_device *hdev, u64 addr, u32 data)
-{
- struct pci_dev *pdev = hdev->pdev;
- ktime_t timeout;
- u32 val;
-
- /* Clear previous status */
- pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, 0);
-
- pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_ADDR, (u32) addr);
- pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_DATA, data);
- pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_CTRL,
- PCI_CONFIG_ELBI_CTRL_WRITE);
-
- timeout = ktime_add_ms(ktime_get(), 10);
- for (;;) {
- pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, &val);
- if (val & PCI_CONFIG_ELBI_STS_MASK)
- break;
- if (ktime_compare(ktime_get(), timeout) > 0) {
- pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS,
- &val);
- break;
- }
- usleep_range(300, 500);
- }
-
- if ((val & PCI_CONFIG_ELBI_STS_MASK) == PCI_CONFIG_ELBI_STS_DONE)
- return 0;
-
- if (val & PCI_CONFIG_ELBI_STS_ERR) {
- dev_err(hdev->dev, "Error writing to ELBI\n");
- return -EIO;
- }
-
- if (!(val & PCI_CONFIG_ELBI_STS_MASK)) {
- dev_err(hdev->dev, "ELBI write didn't finish in time\n");
- return -EIO;
- }
-
- dev_err(hdev->dev, "ELBI write has undefined bits in status\n");
- return -EIO;
-}
-
-/*
- * goya_iatu_write - iatu write routine
- *
- * @hdev: pointer to hl_device structure
- *
- */
-static int goya_iatu_write(struct hl_device *hdev, u32 addr, u32 data)
-{
- u32 dbi_offset;
- int rc;
-
- dbi_offset = addr & 0xFFF;
-
- rc = goya_elbi_write(hdev, CFG_BASE + mmPCIE_AUX_DBI, 0x00300000);
- rc |= goya_elbi_write(hdev, mmPCIE_DBI_BASE + dbi_offset, data);
-
- if (rc)
- return -EIO;
+ (CFG_BASE - SRAM_BASE_ADDR);
return 0;
}
-static void goya_reset_link_through_bridge(struct hl_device *hdev)
-{
- struct pci_dev *pdev = hdev->pdev;
- struct pci_dev *parent_port;
- u16 val;
-
- parent_port = pdev->bus->self;
- pci_read_config_word(parent_port, PCI_BRIDGE_CONTROL, &val);
- val |= PCI_BRIDGE_CTL_BUS_RESET;
- pci_write_config_word(parent_port, PCI_BRIDGE_CONTROL, val);
- ssleep(1);
-
- val &= ~(PCI_BRIDGE_CTL_BUS_RESET);
- pci_write_config_word(parent_port, PCI_BRIDGE_CONTROL, val);
- ssleep(3);
-}
-
-/*
- * goya_set_ddr_bar_base - set DDR bar to map specific device address
- *
- * @hdev: pointer to hl_device structure
- * @addr: address in DDR. Must be aligned to DDR bar size
- *
- * This function configures the iATU so that the DDR bar will start at the
- * specified addr.
- *
- */
-static int goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr)
+static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr)
{
struct goya_device *goya = hdev->asic_specific;
+ u64 old_addr = addr;
int rc;
if ((goya) && (goya->ddr_bar_cur_addr == addr))
- return 0;
+ return old_addr;
/* Inbound Region 1 - Bar 4 - Point to DDR */
- rc = goya_iatu_write(hdev, 0x314, lower_32_bits(addr));
- rc |= goya_iatu_write(hdev, 0x318, upper_32_bits(addr));
- rc |= goya_iatu_write(hdev, 0x300, 0);
- /* Enable + Bar match + match enable + Bar 4 */
- rc |= goya_iatu_write(hdev, 0x304, 0xC0080400);
-
- /* Return the DBI window to the default location */
- rc |= goya_elbi_write(hdev, CFG_BASE + mmPCIE_AUX_DBI, 0);
- rc |= goya_elbi_write(hdev, CFG_BASE + mmPCIE_AUX_DBI_32, 0);
-
- if (rc) {
- dev_err(hdev->dev, "failed to map DDR bar to 0x%08llx\n", addr);
- return -EIO;
- }
+ rc = hl_pci_set_dram_bar_base(hdev, 1, 4, addr);
+ if (rc)
+ return U64_MAX;
- if (goya)
+ if (goya) {
+ old_addr = goya->ddr_bar_cur_addr;
goya->ddr_bar_cur_addr = addr;
+ }
- return 0;
+ return old_addr;
}
/*
@@ -603,40 +420,8 @@ static int goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr)
*/
static int goya_init_iatu(struct hl_device *hdev)
{
- int rc;
-
- /* Inbound Region 0 - Bar 0 - Point to SRAM_BASE_ADDR */
- rc = goya_iatu_write(hdev, 0x114, lower_32_bits(SRAM_BASE_ADDR));
- rc |= goya_iatu_write(hdev, 0x118, upper_32_bits(SRAM_BASE_ADDR));
- rc |= goya_iatu_write(hdev, 0x100, 0);
- /* Enable + Bar match + match enable */
- rc |= goya_iatu_write(hdev, 0x104, 0xC0080000);
-
- /* Inbound Region 1 - Bar 4 - Point to DDR */
- rc |= goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE);
-
- /* Outbound Region 0 - Point to Host */
- rc |= goya_iatu_write(hdev, 0x008, lower_32_bits(HOST_PHYS_BASE));
- rc |= goya_iatu_write(hdev, 0x00C, upper_32_bits(HOST_PHYS_BASE));
- rc |= goya_iatu_write(hdev, 0x010,
- lower_32_bits(HOST_PHYS_BASE + HOST_PHYS_SIZE - 1));
- rc |= goya_iatu_write(hdev, 0x014, 0);
- rc |= goya_iatu_write(hdev, 0x018, 0);
- rc |= goya_iatu_write(hdev, 0x020,
- upper_32_bits(HOST_PHYS_BASE + HOST_PHYS_SIZE - 1));
- /* Increase region size */
- rc |= goya_iatu_write(hdev, 0x000, 0x00002000);
- /* Enable */
- rc |= goya_iatu_write(hdev, 0x004, 0x80000000);
-
- /* Return the DBI window to the default location */
- rc |= goya_elbi_write(hdev, CFG_BASE + mmPCIE_AUX_DBI, 0);
- rc |= goya_elbi_write(hdev, CFG_BASE + mmPCIE_AUX_DBI_32, 0);
-
- if (rc)
- return -EIO;
-
- return 0;
+ return hl_pci_init_iatu(hdev, SRAM_BASE_ADDR, DRAM_PHYS_BASE,
+ HOST_PHYS_BASE, HOST_PHYS_SIZE);
}
/*
@@ -682,52 +467,9 @@ static int goya_early_init(struct hl_device *hdev)
prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID);
- /* set DMA mask for GOYA */
- rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(39));
- if (rc) {
- dev_warn(hdev->dev, "Unable to set pci dma mask to 39 bits\n");
- rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(hdev->dev,
- "Unable to set pci dma mask to 32 bits\n");
- return rc;
- }
- }
-
- rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
- if (rc) {
- dev_warn(hdev->dev,
- "Unable to set pci consistent dma mask to 39 bits\n");
- rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(hdev->dev,
- "Unable to set pci consistent dma mask to 32 bits\n");
- return rc;
- }
- }
-
- if (hdev->reset_pcilink)
- goya_reset_link_through_bridge(hdev);
-
- rc = pci_enable_device_mem(pdev);
- if (rc) {
- dev_err(hdev->dev, "can't enable PCI device\n");
+ rc = hl_pci_init(hdev, 39);
+ if (rc)
return rc;
- }
-
- pci_set_master(pdev);
-
- rc = goya_init_iatu(hdev);
- if (rc) {
- dev_err(hdev->dev, "Failed to initialize iATU\n");
- goto disable_device;
- }
-
- rc = goya_pci_bars_map(hdev);
- if (rc) {
- dev_err(hdev->dev, "Failed to initialize PCI BARS\n");
- goto disable_device;
- }
if (!hdev->pldm) {
val = RREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS);
@@ -737,12 +479,6 @@ static int goya_early_init(struct hl_device *hdev)
}
return 0;
-
-disable_device:
- pci_clear_master(pdev);
- pci_disable_device(pdev);
-
- return rc;
}
/*
@@ -755,14 +491,33 @@ disable_device:
*/
static int goya_early_fini(struct hl_device *hdev)
{
- goya_pci_bars_unmap(hdev);
-
- pci_clear_master(hdev->pdev);
- pci_disable_device(hdev->pdev);
+ hl_pci_fini(hdev);
return 0;
}
+static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)
+{
+ /* mask to zero the MMBP and ASID bits */
+ WREG32_AND(reg, ~0x7FF);
+ WREG32_OR(reg, asid);
+}
+
+static void goya_qman0_set_security(struct hl_device *hdev, bool secure)
+{
+ struct goya_device *goya = hdev->asic_specific;
+
+ if (!(goya->hw_cap_initialized & HW_CAP_MMU))
+ return;
+
+ if (secure)
+ WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_FULLY_TRUSTED);
+ else
+ WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_PARTLY_TRUSTED);
+
+ RREG32(mmDMA_QM_0_GLBL_PROT);
+}
+
/*
* goya_fetch_psoc_frequency - Fetch PSOC frequency values
*
@@ -779,20 +534,12 @@ static void goya_fetch_psoc_frequency(struct hl_device *hdev)
prop->psoc_pci_pll_div_factor = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1);
}
-/*
- * goya_late_init - GOYA late initialization code
- *
- * @hdev: pointer to hl_device structure
- *
- * Get ArmCP info and send message to CPU to enable PCI access
- */
-static int goya_late_init(struct hl_device *hdev)
+int goya_late_init(struct hl_device *hdev)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
- struct goya_device *goya = hdev->asic_specific;
int rc;
- rc = goya->armcp_info_get(hdev);
+ rc = goya_armcp_info_get(hdev);
if (rc) {
dev_err(hdev->dev, "Failed to get armcp info\n");
return rc;
@@ -804,7 +551,7 @@ static int goya_late_init(struct hl_device *hdev)
*/
WREG32(mmMMU_LOG2_DDR_SIZE, ilog2(prop->dram_size));
- rc = goya_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS);
+ rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS);
if (rc) {
dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
return rc;
@@ -830,7 +577,7 @@ static int goya_late_init(struct hl_device *hdev)
return 0;
disable_pci_access:
- goya_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
+ hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
return rc;
}
@@ -879,9 +626,6 @@ static int goya_sw_init(struct hl_device *hdev)
if (!goya)
return -ENOMEM;
- goya->test_cpu_queue = goya_test_cpu_queue;
- goya->armcp_info_get = goya_armcp_info_get;
-
/* according to goya_init_iatu */
goya->ddr_bar_cur_addr = DRAM_PHYS_BASE;
@@ -901,45 +645,43 @@ static int goya_sw_init(struct hl_device *hdev)
}
hdev->cpu_accessible_dma_mem =
- hdev->asic_funcs->dma_alloc_coherent(hdev,
- CPU_ACCESSIBLE_MEM_SIZE,
+ hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
+ HL_CPU_ACCESSIBLE_MEM_SIZE,
&hdev->cpu_accessible_dma_address,
GFP_KERNEL | __GFP_ZERO);
if (!hdev->cpu_accessible_dma_mem) {
- dev_err(hdev->dev,
- "failed to allocate %d of dma memory for CPU accessible memory space\n",
- CPU_ACCESSIBLE_MEM_SIZE);
rc = -ENOMEM;
goto free_dma_pool;
}
- hdev->cpu_accessible_dma_pool = gen_pool_create(CPU_PKT_SHIFT, -1);
+ hdev->cpu_accessible_dma_pool = gen_pool_create(HL_CPU_PKT_SHIFT, -1);
if (!hdev->cpu_accessible_dma_pool) {
dev_err(hdev->dev,
"Failed to create CPU accessible DMA pool\n");
rc = -ENOMEM;
- goto free_cpu_pq_dma_mem;
+ goto free_cpu_dma_mem;
}
rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
(uintptr_t) hdev->cpu_accessible_dma_mem,
- CPU_ACCESSIBLE_MEM_SIZE, -1);
+ HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
if (rc) {
dev_err(hdev->dev,
"Failed to add memory to CPU accessible DMA pool\n");
rc = -EFAULT;
- goto free_cpu_pq_pool;
+ goto free_cpu_accessible_dma_pool;
}
spin_lock_init(&goya->hw_queues_lock);
return 0;
-free_cpu_pq_pool:
+free_cpu_accessible_dma_pool:
gen_pool_destroy(hdev->cpu_accessible_dma_pool);
-free_cpu_pq_dma_mem:
- hdev->asic_funcs->dma_free_coherent(hdev, CPU_ACCESSIBLE_MEM_SIZE,
+free_cpu_dma_mem:
+ hdev->asic_funcs->asic_dma_free_coherent(hdev,
+ HL_CPU_ACCESSIBLE_MEM_SIZE,
hdev->cpu_accessible_dma_mem,
hdev->cpu_accessible_dma_address);
free_dma_pool:
@@ -962,7 +704,8 @@ static int goya_sw_fini(struct hl_device *hdev)
gen_pool_destroy(hdev->cpu_accessible_dma_pool);
- hdev->asic_funcs->dma_free_coherent(hdev, CPU_ACCESSIBLE_MEM_SIZE,
+ hdev->asic_funcs->asic_dma_free_coherent(hdev,
+ HL_CPU_ACCESSIBLE_MEM_SIZE,
hdev->cpu_accessible_dma_mem,
hdev->cpu_accessible_dma_address);
@@ -1056,11 +799,10 @@ static void goya_init_dma_ch(struct hl_device *hdev, int dma_id)
* Initialize the H/W registers of the QMAN DMA channels
*
*/
-static void goya_init_dma_qmans(struct hl_device *hdev)
+void goya_init_dma_qmans(struct hl_device *hdev)
{
struct goya_device *goya = hdev->asic_specific;
struct hl_hw_queue *q;
- dma_addr_t bus_address;
int i;
if (goya->hw_cap_initialized & HW_CAP_DMA)
@@ -1069,10 +811,7 @@ static void goya_init_dma_qmans(struct hl_device *hdev)
q = &hdev->kernel_queues[0];
for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++, q++) {
- bus_address = q->bus_address +
- hdev->asic_prop.host_phys_base_address;
-
- goya_init_dma_qman(hdev, i, bus_address);
+ goya_init_dma_qman(hdev, i, q->bus_address);
goya_init_dma_ch(hdev, i);
}
@@ -1209,11 +948,10 @@ static int goya_stop_external_queues(struct hl_device *hdev)
* Returns 0 on success
*
*/
-static int goya_init_cpu_queues(struct hl_device *hdev)
+int goya_init_cpu_queues(struct hl_device *hdev)
{
struct goya_device *goya = hdev->asic_specific;
struct hl_eq *eq;
- dma_addr_t bus_address;
u32 status;
struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ];
int err;
@@ -1226,23 +964,22 @@ static int goya_init_cpu_queues(struct hl_device *hdev)
eq = &hdev->event_queue;
- bus_address = cpu_pq->bus_address +
- hdev->asic_prop.host_phys_base_address;
- WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_0, lower_32_bits(bus_address));
- WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_1, upper_32_bits(bus_address));
+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_0,
+ lower_32_bits(cpu_pq->bus_address));
+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_1,
+ upper_32_bits(cpu_pq->bus_address));
- bus_address = eq->bus_address + hdev->asic_prop.host_phys_base_address;
- WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_2, lower_32_bits(bus_address));
- WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_3, upper_32_bits(bus_address));
+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_2, lower_32_bits(eq->bus_address));
+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_3, upper_32_bits(eq->bus_address));
- bus_address = hdev->cpu_accessible_dma_address +
- hdev->asic_prop.host_phys_base_address;
- WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_8, lower_32_bits(bus_address));
- WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_9, upper_32_bits(bus_address));
+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_8,
+ lower_32_bits(hdev->cpu_accessible_dma_address));
+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_9,
+ upper_32_bits(hdev->cpu_accessible_dma_address));
WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_5, HL_QUEUE_SIZE_IN_BYTES);
WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_4, HL_EQ_SIZE_IN_BYTES);
- WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_10, CPU_ACCESSIBLE_MEM_SIZE);
+ WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_10, HL_CPU_ACCESSIBLE_MEM_SIZE);
/* Used for EQ CI */
WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_6, 0);
@@ -1695,6 +1432,8 @@ static void goya_init_golden_registers(struct hl_device *hdev)
*/
WREG32(mmDMA_CH_1_CFG0, 0x0fff00F0);
+ WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020);
+
goya->hw_cap_initialized |= HW_CAP_GOLDEN;
}
@@ -1788,7 +1527,7 @@ static void goya_init_mme_cmdq(struct hl_device *hdev)
WREG32(mmMME_CMDQ_GLBL_CFG0, CMDQ_MME_ENABLE);
}
-static void goya_init_mme_qmans(struct hl_device *hdev)
+void goya_init_mme_qmans(struct hl_device *hdev)
{
struct goya_device *goya = hdev->asic_specific;
u32 so_base_lo, so_base_hi;
@@ -1895,7 +1634,7 @@ static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id)
WREG32(mmTPC0_CMDQ_GLBL_CFG0 + reg_off, CMDQ_TPC_ENABLE);
}
-static void goya_init_tpc_qmans(struct hl_device *hdev)
+void goya_init_tpc_qmans(struct hl_device *hdev)
{
struct goya_device *goya = hdev->asic_specific;
u32 so_base_lo, so_base_hi;
@@ -2222,10 +1961,10 @@ static int goya_enable_msix(struct hl_device *hdev)
}
}
- irq = pci_irq_vector(hdev->pdev, EVENT_QUEUE_MSIX_IDX);
+ irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
rc = request_irq(irq, hl_irq_handler_eq, 0,
- goya_irq_name[EVENT_QUEUE_MSIX_IDX],
+ goya_irq_name[GOYA_EVENT_QUEUE_MSIX_IDX],
&hdev->event_queue);
if (rc) {
dev_err(hdev->dev, "Failed to request IRQ %d", irq);
@@ -2256,7 +1995,7 @@ static void goya_sync_irqs(struct hl_device *hdev)
for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
synchronize_irq(pci_irq_vector(hdev->pdev, i));
- synchronize_irq(pci_irq_vector(hdev->pdev, EVENT_QUEUE_MSIX_IDX));
+ synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX));
}
static void goya_disable_msix(struct hl_device *hdev)
@@ -2269,7 +2008,7 @@ static void goya_disable_msix(struct hl_device *hdev)
goya_sync_irqs(hdev);
- irq = pci_irq_vector(hdev->pdev, EVENT_QUEUE_MSIX_IDX);
+ irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
free_irq(irq, &hdev->event_queue);
for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
@@ -2329,67 +2068,45 @@ static void goya_halt_engines(struct hl_device *hdev, bool hard_reset)
}
/*
- * goya_push_fw_to_device - Push FW code to device
- *
- * @hdev: pointer to hl_device structure
+ * goya_push_uboot_to_device() - Push u-boot FW code to device.
+ * @hdev: Pointer to hl_device structure.
*
- * Copy fw code from firmware file to device memory.
- * Returns 0 on success
+ * Copy u-boot fw code from firmware file to SRAM BAR.
*
+ * Return: 0 on success, non-zero for failure.
*/
-static int goya_push_fw_to_device(struct hl_device *hdev, const char *fw_name,
- void __iomem *dst)
+static int goya_push_uboot_to_device(struct hl_device *hdev)
{
- const struct firmware *fw;
- const u64 *fw_data;
- size_t fw_size, i;
- int rc;
-
- rc = request_firmware(&fw, fw_name, hdev->dev);
-
- if (rc) {
- dev_err(hdev->dev, "Failed to request %s\n", fw_name);
- goto out;
- }
-
- fw_size = fw->size;
- if ((fw_size % 4) != 0) {
- dev_err(hdev->dev, "illegal %s firmware size %zu\n",
- fw_name, fw_size);
- rc = -EINVAL;
- goto out;
- }
-
- dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
-
- fw_data = (const u64 *) fw->data;
+ char fw_name[200];
+ void __iomem *dst;
- if ((fw->size % 8) != 0)
- fw_size -= 8;
+ snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-u-boot.bin");
+ dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + UBOOT_FW_OFFSET;
- for (i = 0 ; i < fw_size ; i += 8, fw_data++, dst += 8) {
- if (!(i & (0x80000 - 1))) {
- dev_dbg(hdev->dev,
- "copied so far %zu out of %zu for %s firmware",
- i, fw_size, fw_name);
- usleep_range(20, 100);
- }
+ return hl_fw_push_fw_to_device(hdev, fw_name, dst);
+}
- writeq(*fw_data, dst);
- }
+/*
+ * goya_push_linux_to_device() - Push LINUX FW code to device.
+ * @hdev: Pointer to hl_device structure.
+ *
+ * Copy LINUX fw code from firmware file to HBM BAR.
+ *
+ * Return: 0 on success, non-zero for failure.
+ */
+static int goya_push_linux_to_device(struct hl_device *hdev)
+{
+ char fw_name[200];
+ void __iomem *dst;
- if ((fw->size % 8) != 0)
- writel(*(const u32 *) fw_data, dst);
+ snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-fit.itb");
+ dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
-out:
- release_firmware(fw);
- return rc;
+ return hl_fw_push_fw_to_device(hdev, fw_name, dst);
}
static int goya_pldm_init_cpu(struct hl_device *hdev)
{
- char fw_name[200];
- void __iomem *dst;
u32 val, unit_rst_val;
int rc;
@@ -2407,15 +2124,11 @@ static int goya_pldm_init_cpu(struct hl_device *hdev)
WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, unit_rst_val);
val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
- snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-u-boot.bin");
- dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + UBOOT_FW_OFFSET;
- rc = goya_push_fw_to_device(hdev, fw_name, dst);
+ rc = goya_push_uboot_to_device(hdev);
if (rc)
return rc;
- snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-fit.itb");
- dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
- rc = goya_push_fw_to_device(hdev, fw_name, dst);
+ rc = goya_push_linux_to_device(hdev);
if (rc)
return rc;
@@ -2477,8 +2190,6 @@ static void goya_read_device_fw_version(struct hl_device *hdev,
static int goya_init_cpu(struct hl_device *hdev, u32 cpu_timeout)
{
struct goya_device *goya = hdev->asic_specific;
- char fw_name[200];
- void __iomem *dst;
u32 status;
int rc;
@@ -2492,11 +2203,10 @@ static int goya_init_cpu(struct hl_device *hdev, u32 cpu_timeout)
* Before pushing u-boot/linux to device, need to set the ddr bar to
* base address of dram
*/
- rc = goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE);
- if (rc) {
+ if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
dev_err(hdev->dev,
"failed to map DDR bar to DRAM base address\n");
- return rc;
+ return -EIO;
}
if (hdev->pldm) {
@@ -2549,6 +2259,11 @@ static int goya_init_cpu(struct hl_device *hdev, u32 cpu_timeout)
"ARM status %d - DDR initialization failed\n",
status);
break;
+ case CPU_BOOT_STATUS_UBOOT_NOT_READY:
+ dev_err(hdev->dev,
+ "ARM status %d - u-boot stopped by user\n",
+ status);
+ break;
default:
dev_err(hdev->dev,
"ARM status %d - Invalid status code\n",
@@ -2570,9 +2285,7 @@ static int goya_init_cpu(struct hl_device *hdev, u32 cpu_timeout)
goto out;
}
- snprintf(fw_name, sizeof(fw_name), "habanalabs/goya/goya-fit.itb");
- dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
- rc = goya_push_fw_to_device(hdev, fw_name, dst);
+ rc = goya_push_linux_to_device(hdev);
if (rc)
return rc;
@@ -2605,7 +2318,39 @@ out:
return 0;
}
-static int goya_mmu_init(struct hl_device *hdev)
+static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
+ u64 phys_addr)
+{
+ u32 status, timeout_usec;
+ int rc;
+
+ if (hdev->pldm)
+ timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
+ else
+ timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
+
+ WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
+ WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT);
+ WREG32(MMU_ASID_BUSY, 0x80000000 | asid);
+
+ rc = hl_poll_timeout(
+ hdev,
+ MMU_ASID_BUSY,
+ status,
+ !(status & 0x80000000),
+ 1000,
+ timeout_usec);
+
+ if (rc) {
+ dev_err(hdev->dev,
+ "Timeout during MMU hop0 config of asid %d\n", asid);
+ return rc;
+ }
+
+ return 0;
+}
+
+int goya_mmu_init(struct hl_device *hdev)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
struct goya_device *goya = hdev->asic_specific;
@@ -2696,12 +2441,12 @@ static int goya_hw_init(struct hl_device *hdev)
* After CPU initialization is finished, change DDR bar mapping inside
* iATU to point to the start address of the MMU page tables
*/
- rc = goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE +
- (MMU_PAGE_TABLES_ADDR & ~(prop->dram_pci_bar_size - 0x1ull)));
- if (rc) {
+ if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE +
+ (MMU_PAGE_TABLES_ADDR &
+ ~(prop->dram_pci_bar_size - 0x1ull))) == U64_MAX) {
dev_err(hdev->dev,
"failed to map DDR bar to MMU page tables\n");
- return rc;
+ return -EIO;
}
rc = goya_mmu_init(hdev);
@@ -2728,28 +2473,16 @@ static int goya_hw_init(struct hl_device *hdev)
goto disable_msix;
}
- /* CPU initialization is finished, we can now move to 48 bit DMA mask */
- rc = pci_set_dma_mask(hdev->pdev, DMA_BIT_MASK(48));
- if (rc) {
- dev_warn(hdev->dev, "Unable to set pci dma mask to 48 bits\n");
- rc = pci_set_dma_mask(hdev->pdev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(hdev->dev,
- "Unable to set pci dma mask to 32 bits\n");
- goto disable_pci_access;
- }
- }
-
- rc = pci_set_consistent_dma_mask(hdev->pdev, DMA_BIT_MASK(48));
- if (rc) {
- dev_warn(hdev->dev,
- "Unable to set pci consistent dma mask to 48 bits\n");
- rc = pci_set_consistent_dma_mask(hdev->pdev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(hdev->dev,
- "Unable to set pci consistent dma mask to 32 bits\n");
+ /*
+ * Check if we managed to set the DMA mask to more then 32 bits. If so,
+ * let's try to increase it again because in Goya we set the initial
+ * dma mask to less then 39 bits so that the allocation of the memory
+ * area for the device's cpu will be under 39 bits
+ */
+ if (hdev->dma_mask > 32) {
+ rc = hl_pci_set_dma_mask(hdev, 48);
+ if (rc)
goto disable_pci_access;
- }
}
/* Perform read from the device to flush all MSI-X configuration */
@@ -2758,7 +2491,7 @@ static int goya_hw_init(struct hl_device *hdev)
return 0;
disable_pci_access:
- goya_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
+ hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
disable_msix:
goya_disable_msix(hdev);
disable_queues:
@@ -2865,7 +2598,7 @@ int goya_suspend(struct hl_device *hdev)
{
int rc;
- rc = goya_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
+ rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
if (rc)
dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
@@ -2893,7 +2626,7 @@ static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
return rc;
}
-static void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
+void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
{
u32 db_reg_offset, db_value;
bool invalid_queue = false;
@@ -2991,13 +2724,23 @@ void goya_flush_pq_write(struct hl_device *hdev, u64 *pq, u64 exp_val)
static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags)
{
- return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags);
+ void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,
+ dma_handle, flags);
+
+ /* Shift to the device's base physical address of host memory */
+ if (kernel_addr)
+ *dma_handle += HOST_PHYS_BASE;
+
+ return kernel_addr;
}
static void goya_dma_free_coherent(struct hl_device *hdev, size_t size,
void *cpu_addr, dma_addr_t dma_handle)
{
- dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle);
+ /* Cancel the device's base physical address of host memory */
+ dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE;
+
+ dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);
}
void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id,
@@ -3060,12 +2803,12 @@ void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id,
static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job)
{
- struct goya_device *goya = hdev->asic_specific;
struct packet_msg_prot *fence_pkt;
u32 *fence_ptr;
dma_addr_t fence_dma_addr;
struct hl_cb *cb;
u32 tmp, timeout;
+ char buf[16] = {};
int rc;
if (hdev->pldm)
@@ -3073,13 +2816,14 @@ static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job)
else
timeout = HL_DEVICE_TIMEOUT_USEC;
- if (!hdev->asic_funcs->is_device_idle(hdev)) {
+ if (!hdev->asic_funcs->is_device_idle(hdev, buf, sizeof(buf))) {
dev_err_ratelimited(hdev->dev,
- "Can't send KMD job on QMAN0 if device is not idle\n");
+ "Can't send KMD job on QMAN0 because %s is busy\n",
+ buf);
return -EBUSY;
}
- fence_ptr = hdev->asic_funcs->dma_pool_zalloc(hdev, 4, GFP_KERNEL,
+ fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
&fence_dma_addr);
if (!fence_ptr) {
dev_err(hdev->dev,
@@ -3089,10 +2833,7 @@ static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job)
*fence_ptr = 0;
- if (goya->hw_cap_initialized & HW_CAP_MMU) {
- WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_FULLY_TRUSTED);
- RREG32(mmDMA_QM_0_GLBL_PROT);
- }
+ goya_qman0_set_security(hdev, true);
/*
* goya cs parser saves space for 2xpacket_msg_prot at end of CB. For
@@ -3110,8 +2851,7 @@ static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job)
(1 << GOYA_PKT_CTL_MB_SHIFT);
fence_pkt->ctl = cpu_to_le32(tmp);
fence_pkt->value = cpu_to_le32(GOYA_QMAN0_FENCE_VAL);
- fence_pkt->addr = cpu_to_le64(fence_dma_addr +
- hdev->asic_prop.host_phys_base_address);
+ fence_pkt->addr = cpu_to_le64(fence_dma_addr);
rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_DMA_0,
job->job_cb_size, cb->bus_address);
@@ -3131,13 +2871,10 @@ static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job)
}
free_fence_ptr:
- hdev->asic_funcs->dma_pool_free(hdev, (void *) fence_ptr,
+ hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
fence_dma_addr);
- if (goya->hw_cap_initialized & HW_CAP_MMU) {
- WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_PARTLY_TRUSTED);
- RREG32(mmDMA_QM_0_GLBL_PROT);
- }
+ goya_qman0_set_security(hdev, false);
return rc;
}
@@ -3146,10 +2883,6 @@ int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
u32 timeout, long *result)
{
struct goya_device *goya = hdev->asic_specific;
- struct armcp_packet *pkt;
- dma_addr_t pkt_dma_addr;
- u32 tmp;
- int rc = 0;
if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) {
if (result)
@@ -3157,74 +2890,8 @@ int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
return 0;
}
- if (len > CPU_CB_SIZE) {
- dev_err(hdev->dev, "Invalid CPU message size of %d bytes\n",
- len);
- return -ENOMEM;
- }
-
- pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len,
- &pkt_dma_addr);
- if (!pkt) {
- dev_err(hdev->dev,
- "Failed to allocate DMA memory for packet to CPU\n");
- return -ENOMEM;
- }
-
- memcpy(pkt, msg, len);
-
- mutex_lock(&hdev->send_cpu_message_lock);
-
- if (hdev->disabled)
- goto out;
-
- if (hdev->device_cpu_disabled) {
- rc = -EIO;
- goto out;
- }
-
- rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_CPU_PQ, len,
- pkt_dma_addr);
- if (rc) {
- dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc);
- goto out;
- }
-
- rc = hl_poll_timeout_memory(hdev, (u64) (uintptr_t) &pkt->fence,
- timeout, &tmp);
-
- hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_CPU_PQ);
-
- if (rc == -ETIMEDOUT) {
- dev_err(hdev->dev, "Timeout while waiting for device CPU\n");
- hdev->device_cpu_disabled = true;
- goto out;
- }
-
- if (tmp == ARMCP_PACKET_FENCE_VAL) {
- u32 ctl = le32_to_cpu(pkt->ctl);
-
- rc = (ctl & ARMCP_PKT_CTL_RC_MASK) >> ARMCP_PKT_CTL_RC_SHIFT;
- if (rc) {
- dev_err(hdev->dev,
- "F/W ERROR %d for CPU packet %d\n",
- rc, (ctl & ARMCP_PKT_CTL_OPCODE_MASK)
- >> ARMCP_PKT_CTL_OPCODE_SHIFT);
- rc = -EINVAL;
- } else if (result) {
- *result = (long) le64_to_cpu(pkt->result);
- }
- } else {
- dev_err(hdev->dev, "CPU packet wrong fence value\n");
- rc = -EINVAL;
- }
-
-out:
- mutex_unlock(&hdev->send_cpu_message_lock);
-
- hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, len, pkt);
-
- return rc;
+ return hl_fw_send_cpu_message(hdev, GOYA_QUEUE_ID_CPU_PQ, msg, len,
+ timeout, result);
}
int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
@@ -3238,7 +2905,7 @@ int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
fence_val = GOYA_QMAN0_FENCE_VAL;
- fence_ptr = hdev->asic_funcs->dma_pool_zalloc(hdev, 4, GFP_KERNEL,
+ fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
&fence_dma_addr);
if (!fence_ptr) {
dev_err(hdev->dev,
@@ -3248,7 +2915,7 @@ int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
*fence_ptr = 0;
- fence_pkt = hdev->asic_funcs->dma_pool_zalloc(hdev,
+ fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev,
sizeof(struct packet_msg_prot),
GFP_KERNEL, &pkt_dma_addr);
if (!fence_pkt) {
@@ -3263,8 +2930,7 @@ int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
(1 << GOYA_PKT_CTL_MB_SHIFT);
fence_pkt->ctl = cpu_to_le32(tmp);
fence_pkt->value = cpu_to_le32(fence_val);
- fence_pkt->addr = cpu_to_le64(fence_dma_addr +
- hdev->asic_prop.host_phys_base_address);
+ fence_pkt->addr = cpu_to_le64(fence_dma_addr);
rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
sizeof(struct packet_msg_prot),
@@ -3292,48 +2958,30 @@ int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
}
free_pkt:
- hdev->asic_funcs->dma_pool_free(hdev, (void *) fence_pkt,
+ hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt,
pkt_dma_addr);
free_fence_ptr:
- hdev->asic_funcs->dma_pool_free(hdev, (void *) fence_ptr,
+ hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
fence_dma_addr);
return rc;
}
int goya_test_cpu_queue(struct hl_device *hdev)
{
- struct armcp_packet test_pkt;
- long result;
- int rc;
-
- /* cpu_queues_enable flag is always checked in send cpu message */
-
- memset(&test_pkt, 0, sizeof(test_pkt));
-
- test_pkt.ctl = cpu_to_le32(ARMCP_PACKET_TEST <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
- test_pkt.value = cpu_to_le64(ARMCP_PACKET_FENCE_VAL);
-
- rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,
- sizeof(test_pkt), HL_DEVICE_TIMEOUT_USEC, &result);
+ struct goya_device *goya = hdev->asic_specific;
- if (!rc) {
- if (result == ARMCP_PACKET_FENCE_VAL)
- dev_info(hdev->dev,
- "queue test on CPU queue succeeded\n");
- else
- dev_err(hdev->dev,
- "CPU queue test failed (0x%08lX)\n", result);
- } else {
- dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);
- }
+ /*
+ * check capability here as send_cpu_message() won't update the result
+ * value if no capability
+ */
+ if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
+ return 0;
- return rc;
+ return hl_fw_test_cpu_queue(hdev);
}
-static int goya_test_queues(struct hl_device *hdev)
+int goya_test_queues(struct hl_device *hdev)
{
- struct goya_device *goya = hdev->asic_specific;
int i, rc, ret_val = 0;
for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
@@ -3343,7 +2991,7 @@ static int goya_test_queues(struct hl_device *hdev)
}
if (hdev->cpu_queues_enable) {
- rc = goya->test_cpu_queue(hdev);
+ rc = goya_test_cpu_queue(hdev);
if (rc)
ret_val = -EINVAL;
}
@@ -3354,57 +3002,68 @@ static int goya_test_queues(struct hl_device *hdev)
static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size,
gfp_t mem_flags, dma_addr_t *dma_handle)
{
+ void *kernel_addr;
+
if (size > GOYA_DMA_POOL_BLK_SIZE)
return NULL;
- return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
+ kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
+
+ /* Shift to the device's base physical address of host memory */
+ if (kernel_addr)
+ *dma_handle += HOST_PHYS_BASE;
+
+ return kernel_addr;
}
static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr,
dma_addr_t dma_addr)
{
- dma_pool_free(hdev->dma_pool, vaddr, dma_addr);
+ /* Cancel the device's base physical address of host memory */
+ dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE;
+
+ dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);
}
-static void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev,
- size_t size, dma_addr_t *dma_handle)
+void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
+ dma_addr_t *dma_handle)
{
- u64 kernel_addr;
-
- /* roundup to CPU_PKT_SIZE */
- size = (size + (CPU_PKT_SIZE - 1)) & CPU_PKT_MASK;
-
- kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);
-
- *dma_handle = hdev->cpu_accessible_dma_address +
- (kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);
-
- return (void *) (uintptr_t) kernel_addr;
+ return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
}
-static void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev,
- size_t size, void *vaddr)
+void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
+ void *vaddr)
{
- /* roundup to CPU_PKT_SIZE */
- size = (size + (CPU_PKT_SIZE - 1)) & CPU_PKT_MASK;
-
- gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,
- size);
+ hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
}
-static int goya_dma_map_sg(struct hl_device *hdev, struct scatterlist *sg,
+static int goya_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl,
int nents, enum dma_data_direction dir)
{
- if (!dma_map_sg(&hdev->pdev->dev, sg, nents, dir))
+ struct scatterlist *sg;
+ int i;
+
+ if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir))
return -ENOMEM;
+ /* Shift to the device's base physical address of host memory */
+ for_each_sg(sgl, sg, nents, i)
+ sg->dma_address += HOST_PHYS_BASE;
+
return 0;
}
-static void goya_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sg,
+static void goya_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl,
int nents, enum dma_data_direction dir)
{
- dma_unmap_sg(&hdev->pdev->dev, sg, nents, dir);
+ struct scatterlist *sg;
+ int i;
+
+ /* Cancel the device's base physical address of host memory */
+ for_each_sg(sgl, sg, nents, i)
+ sg->dma_address -= HOST_PHYS_BASE;
+
+ dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir);
}
u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt)
@@ -3554,31 +3213,29 @@ static int goya_validate_dma_pkt_host(struct hl_device *hdev,
return -EFAULT;
}
- if (parser->ctx_id != HL_KERNEL_ASID_ID) {
- if (sram_addr) {
- if (!hl_mem_area_inside_range(device_memory_addr,
- le32_to_cpu(user_dma_pkt->tsize),
- hdev->asic_prop.sram_user_base_address,
- hdev->asic_prop.sram_end_address)) {
+ if (sram_addr) {
+ if (!hl_mem_area_inside_range(device_memory_addr,
+ le32_to_cpu(user_dma_pkt->tsize),
+ hdev->asic_prop.sram_user_base_address,
+ hdev->asic_prop.sram_end_address)) {
+
+ dev_err(hdev->dev,
+ "SRAM address 0x%llx + 0x%x is invalid\n",
+ device_memory_addr,
+ user_dma_pkt->tsize);
+ return -EFAULT;
+ }
+ } else {
+ if (!hl_mem_area_inside_range(device_memory_addr,
+ le32_to_cpu(user_dma_pkt->tsize),
+ hdev->asic_prop.dram_user_base_address,
+ hdev->asic_prop.dram_end_address)) {
- dev_err(hdev->dev,
- "SRAM address 0x%llx + 0x%x is invalid\n",
- device_memory_addr,
- user_dma_pkt->tsize);
- return -EFAULT;
- }
- } else {
- if (!hl_mem_area_inside_range(device_memory_addr,
- le32_to_cpu(user_dma_pkt->tsize),
- hdev->asic_prop.dram_user_base_address,
- hdev->asic_prop.dram_end_address)) {
-
- dev_err(hdev->dev,
- "DRAM address 0x%llx + 0x%x is invalid\n",
- device_memory_addr,
- user_dma_pkt->tsize);
- return -EFAULT;
- }
+ dev_err(hdev->dev,
+ "DRAM address 0x%llx + 0x%x is invalid\n",
+ device_memory_addr,
+ user_dma_pkt->tsize);
+ return -EFAULT;
}
}
@@ -3956,8 +3613,6 @@ static int goya_patch_dma_packet(struct hl_device *hdev,
new_dma_pkt->ctl = cpu_to_le32(ctl);
new_dma_pkt->tsize = cpu_to_le32((u32) len);
- dma_addr += hdev->asic_prop.host_phys_base_address;
-
if (dir == DMA_TO_DEVICE) {
new_dma_pkt->src_addr = cpu_to_le64(dma_addr);
new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr);
@@ -4208,36 +3863,35 @@ free_userptr:
return rc;
}
-static int goya_parse_cb_no_ext_quque(struct hl_device *hdev,
+static int goya_parse_cb_no_ext_queue(struct hl_device *hdev,
struct hl_cs_parser *parser)
{
struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
struct goya_device *goya = hdev->asic_specific;
- if (!(goya->hw_cap_initialized & HW_CAP_MMU)) {
- /* For internal queue jobs, just check if cb address is valid */
- if (hl_mem_area_inside_range(
- (u64) (uintptr_t) parser->user_cb,
- parser->user_cb_size,
- asic_prop->sram_user_base_address,
- asic_prop->sram_end_address))
- return 0;
+ if (goya->hw_cap_initialized & HW_CAP_MMU)
+ return 0;
- if (hl_mem_area_inside_range(
- (u64) (uintptr_t) parser->user_cb,
- parser->user_cb_size,
- asic_prop->dram_user_base_address,
- asic_prop->dram_end_address))
- return 0;
+ /* For internal queue jobs, just check if CB address is valid */
+ if (hl_mem_area_inside_range(
+ (u64) (uintptr_t) parser->user_cb,
+ parser->user_cb_size,
+ asic_prop->sram_user_base_address,
+ asic_prop->sram_end_address))
+ return 0;
- dev_err(hdev->dev,
- "Internal CB address %px + 0x%x is not in SRAM nor in DRAM\n",
- parser->user_cb, parser->user_cb_size);
+ if (hl_mem_area_inside_range(
+ (u64) (uintptr_t) parser->user_cb,
+ parser->user_cb_size,
+ asic_prop->dram_user_base_address,
+ asic_prop->dram_end_address))
+ return 0;
- return -EFAULT;
- }
+ dev_err(hdev->dev,
+ "Internal CB address %px + 0x%x is not in SRAM nor in DRAM\n",
+ parser->user_cb, parser->user_cb_size);
- return 0;
+ return -EFAULT;
}
int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
@@ -4245,9 +3899,9 @@ int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
struct goya_device *goya = hdev->asic_specific;
if (!parser->ext_queue)
- return goya_parse_cb_no_ext_quque(hdev, parser);
+ return goya_parse_cb_no_ext_queue(hdev, parser);
- if ((goya->hw_cap_initialized & HW_CAP_MMU) && parser->use_virt_addr)
+ if (goya->hw_cap_initialized & HW_CAP_MMU)
return goya_parse_cb_mmu(hdev, parser);
else
return goya_parse_cb_no_mmu(hdev, parser);
@@ -4278,12 +3932,12 @@ void goya_add_end_of_cb_packets(u64 kernel_address, u32 len, u64 cq_addr,
cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF);
}
-static void goya_update_eq_ci(struct hl_device *hdev, u32 val)
+void goya_update_eq_ci(struct hl_device *hdev, u32 val)
{
WREG32(mmPSOC_GLOBAL_CONF_SCRATCHPAD_6, val);
}
-static void goya_restore_phase_topology(struct hl_device *hdev)
+void goya_restore_phase_topology(struct hl_device *hdev)
{
int i, num_of_sob_in_longs, num_of_mon_in_longs;
@@ -4320,6 +3974,7 @@ static void goya_restore_phase_topology(struct hl_device *hdev)
static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u64 ddr_bar_addr;
int rc = 0;
if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
@@ -4337,15 +3992,16 @@ static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
u64 bar_base_addr = DRAM_PHYS_BASE +
(addr & ~(prop->dram_pci_bar_size - 0x1ull));
- rc = goya_set_ddr_bar_base(hdev, bar_base_addr);
- if (!rc) {
+ ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
+ if (ddr_bar_addr != U64_MAX) {
*val = readl(hdev->pcie_bar[DDR_BAR_ID] +
(addr - bar_base_addr));
- rc = goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE +
- (MMU_PAGE_TABLES_ADDR &
- ~(prop->dram_pci_bar_size - 0x1ull)));
+ ddr_bar_addr = goya_set_ddr_bar_base(hdev,
+ ddr_bar_addr);
}
+ if (ddr_bar_addr == U64_MAX)
+ rc = -EIO;
} else {
rc = -EFAULT;
}
@@ -4370,6 +4026,7 @@ static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u64 ddr_bar_addr;
int rc = 0;
if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
@@ -4387,15 +4044,16 @@ static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)
u64 bar_base_addr = DRAM_PHYS_BASE +
(addr & ~(prop->dram_pci_bar_size - 0x1ull));
- rc = goya_set_ddr_bar_base(hdev, bar_base_addr);
- if (!rc) {
+ ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
+ if (ddr_bar_addr != U64_MAX) {
writel(val, hdev->pcie_bar[DDR_BAR_ID] +
(addr - bar_base_addr));
- rc = goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE +
- (MMU_PAGE_TABLES_ADDR &
- ~(prop->dram_pci_bar_size - 0x1ull)));
+ ddr_bar_addr = goya_set_ddr_bar_base(hdev,
+ ddr_bar_addr);
}
+ if (ddr_bar_addr == U64_MAX)
+ rc = -EIO;
} else {
rc = -EFAULT;
}
@@ -4407,6 +4065,9 @@ static u64 goya_read_pte(struct hl_device *hdev, u64 addr)
{
struct goya_device *goya = hdev->asic_specific;
+ if (hdev->hard_reset_pending)
+ return U64_MAX;
+
return readq(hdev->pcie_bar[DDR_BAR_ID] +
(addr - goya->ddr_bar_cur_addr));
}
@@ -4415,6 +4076,9 @@ static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val)
{
struct goya_device *goya = hdev->asic_specific;
+ if (hdev->hard_reset_pending)
+ return;
+
writeq(val, hdev->pcie_bar[DDR_BAR_ID] +
(addr - goya->ddr_bar_cur_addr));
}
@@ -4604,8 +4268,8 @@ static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr,
pkt->armcp_pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
ARMCP_PKT_CTL_OPCODE_SHIFT);
- rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
- total_pkt_size, HL_DEVICE_TIMEOUT_USEC, &result);
+ rc = goya_send_cpu_message(hdev, (u32 *) pkt, total_pkt_size,
+ HL_DEVICE_TIMEOUT_USEC, &result);
if (rc)
dev_err(hdev->dev, "failed to unmask IRQ array\n");
@@ -4621,8 +4285,8 @@ static int goya_soft_reset_late_init(struct hl_device *hdev)
* Unmask all IRQs since some could have been received
* during the soft reset
*/
- return goya_unmask_irq_arr(hdev, goya_non_fatal_events,
- sizeof(goya_non_fatal_events));
+ return goya_unmask_irq_arr(hdev, goya_all_events,
+ sizeof(goya_all_events));
}
static int goya_unmask_irq(struct hl_device *hdev, u16 event_type)
@@ -4637,7 +4301,7 @@ static int goya_unmask_irq(struct hl_device *hdev, u16 event_type)
ARMCP_PKT_CTL_OPCODE_SHIFT);
pkt.value = cpu_to_le64(event_type);
- rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+ rc = goya_send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
HL_DEVICE_TIMEOUT_USEC, &result);
if (rc)
@@ -4758,7 +4422,6 @@ static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u32 size,
u64 val, bool is_dram)
{
struct packet_lin_dma *lin_dma_pkt;
- struct hl_cs_parser parser;
struct hl_cs_job *job;
u32 cb_size, ctl;
struct hl_cb *cb;
@@ -4798,36 +4461,16 @@ static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u32 size,
job->user_cb->cs_cnt++;
job->user_cb_size = cb_size;
job->hw_queue_id = GOYA_QUEUE_ID_DMA_0;
+ job->patched_cb = job->user_cb;
+ job->job_cb_size = job->user_cb_size +
+ sizeof(struct packet_msg_prot) * 2;
hl_debugfs_add_job(hdev, job);
- parser.ctx_id = HL_KERNEL_ASID_ID;
- parser.cs_sequence = 0;
- parser.job_id = job->id;
- parser.hw_queue_id = job->hw_queue_id;
- parser.job_userptr_list = &job->userptr_list;
- parser.user_cb = job->user_cb;
- parser.user_cb_size = job->user_cb_size;
- parser.ext_queue = job->ext_queue;
- parser.use_virt_addr = hdev->mmu_enable;
-
- rc = hdev->asic_funcs->cs_parser(hdev, &parser);
- if (rc) {
- dev_err(hdev->dev, "Failed to parse kernel CB\n");
- goto free_job;
- }
-
- job->patched_cb = parser.patched_cb;
- job->job_cb_size = parser.patched_cb_size;
- job->patched_cb->cs_cnt++;
-
rc = goya_send_job_on_qman0(hdev, job);
- job->patched_cb->cs_cnt--;
hl_cb_put(job->patched_cb);
-free_job:
- hl_userptr_delete_list(hdev, &job->userptr_list);
hl_debugfs_remove_job(hdev, job);
kfree(job);
cb->cs_cnt--;
@@ -4839,7 +4482,7 @@ release_cb:
return rc;
}
-static int goya_context_switch(struct hl_device *hdev, u32 asid)
+int goya_context_switch(struct hl_device *hdev, u32 asid)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
u64 addr = prop->sram_base_address;
@@ -4853,12 +4496,13 @@ static int goya_context_switch(struct hl_device *hdev, u32 asid)
return rc;
}
+ WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020);
goya_mmu_prepare(hdev, asid);
return 0;
}
-static int goya_mmu_clear_pgt_range(struct hl_device *hdev)
+int goya_mmu_clear_pgt_range(struct hl_device *hdev)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
struct goya_device *goya = hdev->asic_specific;
@@ -4872,7 +4516,7 @@ static int goya_mmu_clear_pgt_range(struct hl_device *hdev)
return goya_memset_device_memory(hdev, addr, size, 0, true);
}
-static int goya_mmu_set_dram_default_page(struct hl_device *hdev)
+int goya_mmu_set_dram_default_page(struct hl_device *hdev)
{
struct goya_device *goya = hdev->asic_specific;
u64 addr = hdev->asic_prop.mmu_dram_default_page_addr;
@@ -4885,7 +4529,7 @@ static int goya_mmu_set_dram_default_page(struct hl_device *hdev)
return goya_memset_device_memory(hdev, addr, size, val, true);
}
-static void goya_mmu_prepare(struct hl_device *hdev, u32 asid)
+void goya_mmu_prepare(struct hl_device *hdev, u32 asid)
{
struct goya_device *goya = hdev->asic_specific;
int i;
@@ -4899,10 +4543,8 @@ static void goya_mmu_prepare(struct hl_device *hdev, u32 asid)
}
/* zero the MMBP and ASID bits and then set the ASID */
- for (i = 0 ; i < GOYA_MMU_REGS_NUM ; i++) {
- WREG32_AND(goya_mmu_regs[i], ~0x7FF);
- WREG32_OR(goya_mmu_regs[i], asid);
- }
+ for (i = 0 ; i < GOYA_MMU_REGS_NUM ; i++)
+ goya_mmu_prepare_reg(hdev, goya_mmu_regs[i], asid);
}
static void goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard)
@@ -4993,107 +4635,29 @@ static void goya_mmu_invalidate_cache_range(struct hl_device *hdev,
"Timeout when waiting for MMU cache invalidation\n");
}
-static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
- u64 phys_addr)
-{
- u32 status, timeout_usec;
- int rc;
-
- if (hdev->pldm)
- timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
- else
- timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
-
- WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
- WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT);
- WREG32(MMU_ASID_BUSY, 0x80000000 | asid);
-
- rc = hl_poll_timeout(
- hdev,
- MMU_ASID_BUSY,
- status,
- !(status & 0x80000000),
- 1000,
- timeout_usec);
-
- if (rc) {
- dev_err(hdev->dev,
- "Timeout during MMU hop0 config of asid %d\n", asid);
- return rc;
- }
-
- return 0;
-}
-
int goya_send_heartbeat(struct hl_device *hdev)
{
struct goya_device *goya = hdev->asic_specific;
- struct armcp_packet hb_pkt;
- long result;
- int rc;
if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
return 0;
- memset(&hb_pkt, 0, sizeof(hb_pkt));
-
- hb_pkt.ctl = cpu_to_le32(ARMCP_PACKET_TEST <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
- hb_pkt.value = cpu_to_le64(ARMCP_PACKET_FENCE_VAL);
-
- rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt,
- sizeof(hb_pkt), HL_DEVICE_TIMEOUT_USEC, &result);
-
- if ((rc) || (result != ARMCP_PACKET_FENCE_VAL))
- rc = -EIO;
-
- return rc;
+ return hl_fw_send_heartbeat(hdev);
}
-static int goya_armcp_info_get(struct hl_device *hdev)
+int goya_armcp_info_get(struct hl_device *hdev)
{
struct goya_device *goya = hdev->asic_specific;
struct asic_fixed_properties *prop = &hdev->asic_prop;
- struct armcp_packet pkt;
- void *armcp_info_cpu_addr;
- dma_addr_t armcp_info_dma_addr;
u64 dram_size;
- long result;
int rc;
if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
return 0;
- armcp_info_cpu_addr =
- hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
- sizeof(struct armcp_info), &armcp_info_dma_addr);
- if (!armcp_info_cpu_addr) {
- dev_err(hdev->dev,
- "Failed to allocate DMA memory for ArmCP info packet\n");
- return -ENOMEM;
- }
-
- memset(armcp_info_cpu_addr, 0, sizeof(struct armcp_info));
-
- memset(&pkt, 0, sizeof(pkt));
-
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_INFO_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
- pkt.addr = cpu_to_le64(armcp_info_dma_addr +
- prop->host_phys_base_address);
- pkt.data_max_size = cpu_to_le32(sizeof(struct armcp_info));
-
- rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
- GOYA_ARMCP_INFO_TIMEOUT, &result);
-
- if (rc) {
- dev_err(hdev->dev,
- "Failed to send armcp info pkt, error %d\n", rc);
- goto out;
- }
-
- memcpy(&prop->armcp_info, armcp_info_cpu_addr,
- sizeof(prop->armcp_info));
+ rc = hl_fw_armcp_info_get(hdev);
+ if (rc)
+ return rc;
dram_size = le64_to_cpu(prop->armcp_info.dram_size);
if (dram_size) {
@@ -5109,32 +4673,10 @@ static int goya_armcp_info_get(struct hl_device *hdev)
prop->dram_end_address = prop->dram_base_address + dram_size;
}
- rc = hl_build_hwmon_channel_info(hdev, prop->armcp_info.sensors);
- if (rc) {
- dev_err(hdev->dev,
- "Failed to build hwmon channel info, error %d\n", rc);
- rc = -EFAULT;
- goto out;
- }
-
-out:
- hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
- sizeof(struct armcp_info), armcp_info_cpu_addr);
-
- return rc;
-}
-
-static void goya_init_clock_gating(struct hl_device *hdev)
-{
-
-}
-
-static void goya_disable_clock_gating(struct hl_device *hdev)
-{
-
+ return 0;
}
-static bool goya_is_device_idle(struct hl_device *hdev)
+static bool goya_is_device_idle(struct hl_device *hdev, char *buf, size_t size)
{
u64 offset, dma_qm_reg, tpc_qm_reg, tpc_cmdq_reg, tpc_cfg_reg;
int i;
@@ -5146,7 +4688,7 @@ static bool goya_is_device_idle(struct hl_device *hdev)
if ((RREG32(dma_qm_reg) & DMA_QM_IDLE_MASK) !=
DMA_QM_IDLE_MASK)
- return false;
+ return HL_ENG_BUSY(buf, size, "DMA%d_QM", i);
}
offset = mmTPC1_QM_GLBL_STS0 - mmTPC0_QM_GLBL_STS0;
@@ -5158,31 +4700,31 @@ static bool goya_is_device_idle(struct hl_device *hdev)
if ((RREG32(tpc_qm_reg) & TPC_QM_IDLE_MASK) !=
TPC_QM_IDLE_MASK)
- return false;
+ return HL_ENG_BUSY(buf, size, "TPC%d_QM", i);
if ((RREG32(tpc_cmdq_reg) & TPC_CMDQ_IDLE_MASK) !=
TPC_CMDQ_IDLE_MASK)
- return false;
+ return HL_ENG_BUSY(buf, size, "TPC%d_CMDQ", i);
if ((RREG32(tpc_cfg_reg) & TPC_CFG_IDLE_MASK) !=
TPC_CFG_IDLE_MASK)
- return false;
+ return HL_ENG_BUSY(buf, size, "TPC%d_CFG", i);
}
if ((RREG32(mmMME_QM_GLBL_STS0) & MME_QM_IDLE_MASK) !=
MME_QM_IDLE_MASK)
- return false;
+ return HL_ENG_BUSY(buf, size, "MME_QM");
if ((RREG32(mmMME_CMDQ_GLBL_STS0) & MME_CMDQ_IDLE_MASK) !=
MME_CMDQ_IDLE_MASK)
- return false;
+ return HL_ENG_BUSY(buf, size, "MME_CMDQ");
if ((RREG32(mmMME_ARCH_STATUS) & MME_ARCH_IDLE_MASK) !=
MME_ARCH_IDLE_MASK)
- return false;
+ return HL_ENG_BUSY(buf, size, "MME_ARCH");
if (RREG32(mmMME_SHADOW_0_STATUS) & MME_SHADOW_IDLE_MASK)
- return false;
+ return HL_ENG_BUSY(buf, size, "MME");
return true;
}
@@ -5210,52 +4752,11 @@ static int goya_get_eeprom_data(struct hl_device *hdev, void *data,
size_t max_size)
{
struct goya_device *goya = hdev->asic_specific;
- struct asic_fixed_properties *prop = &hdev->asic_prop;
- struct armcp_packet pkt;
- void *eeprom_info_cpu_addr;
- dma_addr_t eeprom_info_dma_addr;
- long result;
- int rc;
if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
return 0;
- eeprom_info_cpu_addr =
- hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
- max_size, &eeprom_info_dma_addr);
- if (!eeprom_info_cpu_addr) {
- dev_err(hdev->dev,
- "Failed to allocate DMA memory for EEPROM info packet\n");
- return -ENOMEM;
- }
-
- memset(eeprom_info_cpu_addr, 0, max_size);
-
- memset(&pkt, 0, sizeof(pkt));
-
- pkt.ctl = cpu_to_le32(ARMCP_PACKET_EEPROM_DATA_GET <<
- ARMCP_PKT_CTL_OPCODE_SHIFT);
- pkt.addr = cpu_to_le64(eeprom_info_dma_addr +
- prop->host_phys_base_address);
- pkt.data_max_size = cpu_to_le32(max_size);
-
- rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
- GOYA_ARMCP_EEPROM_TIMEOUT, &result);
-
- if (rc) {
- dev_err(hdev->dev,
- "Failed to send armcp EEPROM pkt, error %d\n", rc);
- goto out;
- }
-
- /* result contains the actual size */
- memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));
-
-out:
- hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, max_size,
- eeprom_info_cpu_addr);
-
- return rc;
+ return hl_fw_get_eeprom_data(hdev, data, max_size);
}
static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev)
@@ -5278,12 +4779,12 @@ static const struct hl_asic_funcs goya_funcs = {
.cb_mmap = goya_cb_mmap,
.ring_doorbell = goya_ring_doorbell,
.flush_pq_write = goya_flush_pq_write,
- .dma_alloc_coherent = goya_dma_alloc_coherent,
- .dma_free_coherent = goya_dma_free_coherent,
+ .asic_dma_alloc_coherent = goya_dma_alloc_coherent,
+ .asic_dma_free_coherent = goya_dma_free_coherent,
.get_int_queue_base = goya_get_int_queue_base,
.test_queues = goya_test_queues,
- .dma_pool_zalloc = goya_dma_pool_zalloc,
- .dma_pool_free = goya_dma_pool_free,
+ .asic_dma_pool_zalloc = goya_dma_pool_zalloc,
+ .asic_dma_pool_free = goya_dma_pool_free,
.cpu_accessible_dma_pool_alloc = goya_cpu_accessible_dma_pool_alloc,
.cpu_accessible_dma_pool_free = goya_cpu_accessible_dma_pool_free,
.hl_dma_unmap_sg = goya_dma_unmap_sg,
@@ -5305,8 +4806,7 @@ static const struct hl_asic_funcs goya_funcs = {
.mmu_invalidate_cache = goya_mmu_invalidate_cache,
.mmu_invalidate_cache_range = goya_mmu_invalidate_cache_range,
.send_heartbeat = goya_send_heartbeat,
- .enable_clock_gating = goya_init_clock_gating,
- .disable_clock_gating = goya_disable_clock_gating,
+ .debug_coresight = goya_debug_coresight,
.is_device_idle = goya_is_device_idle,
.soft_reset_late_init = goya_soft_reset_late_init,
.hw_queues_lock = goya_hw_queues_lock,
@@ -5314,7 +4814,12 @@ static const struct hl_asic_funcs goya_funcs = {
.get_pci_id = goya_get_pci_id,
.get_eeprom_data = goya_get_eeprom_data,
.send_cpu_message = goya_send_cpu_message,
- .get_hw_state = goya_get_hw_state
+ .get_hw_state = goya_get_hw_state,
+ .pci_bars_map = goya_pci_bars_map,
+ .set_dram_bar_base = goya_set_ddr_bar_base,
+ .init_iatu = goya_init_iatu,
+ .rreg = hl_rreg,
+ .wreg = hl_wreg
};
/*
diff --git a/drivers/misc/habanalabs/goya/goyaP.h b/drivers/misc/habanalabs/goya/goyaP.h
index 830551b6b062..14e216cb3668 100644
--- a/drivers/misc/habanalabs/goya/goyaP.h
+++ b/drivers/misc/habanalabs/goya/goyaP.h
@@ -39,9 +39,13 @@
#error "Number of MSIX interrupts must be smaller or equal to GOYA_MSIX_ENTRIES"
#endif
-#define QMAN_FENCE_TIMEOUT_USEC 10000 /* 10 ms */
+#define QMAN_FENCE_TIMEOUT_USEC 10000 /* 10 ms */
-#define QMAN_STOP_TIMEOUT_USEC 100000 /* 100 ms */
+#define QMAN_STOP_TIMEOUT_USEC 100000 /* 100 ms */
+
+#define CORESIGHT_TIMEOUT_USEC 100000 /* 100 ms */
+
+#define GOYA_CPU_TIMEOUT_USEC 10000000 /* 10s */
#define TPC_ENABLED_MASK 0xFF
@@ -49,19 +53,14 @@
#define MAX_POWER_DEFAULT 200000 /* 200W */
-#define GOYA_ARMCP_INFO_TIMEOUT 10000000 /* 10s */
-#define GOYA_ARMCP_EEPROM_TIMEOUT 10000000 /* 10s */
-
#define DRAM_PHYS_DEFAULT_SIZE 0x100000000ull /* 4GB */
/* DRAM Memory Map */
#define CPU_FW_IMAGE_SIZE 0x10000000 /* 256MB */
-#define MMU_PAGE_TABLES_SIZE 0x0DE00000 /* 222MB */
+#define MMU_PAGE_TABLES_SIZE 0x0FC00000 /* 252MB */
#define MMU_DRAM_DEFAULT_PAGE_SIZE 0x00200000 /* 2MB */
#define MMU_CACHE_MNG_SIZE 0x00001000 /* 4KB */
-#define CPU_PQ_PKT_SIZE 0x00001000 /* 4KB */
-#define CPU_PQ_DATA_SIZE 0x01FFE000 /* 32MB - 8KB */
#define CPU_FW_IMAGE_ADDR DRAM_PHYS_BASE
#define MMU_PAGE_TABLES_ADDR (CPU_FW_IMAGE_ADDR + CPU_FW_IMAGE_SIZE)
@@ -69,13 +68,13 @@
MMU_PAGE_TABLES_SIZE)
#define MMU_CACHE_MNG_ADDR (MMU_DRAM_DEFAULT_PAGE_ADDR + \
MMU_DRAM_DEFAULT_PAGE_SIZE)
-#define CPU_PQ_PKT_ADDR (MMU_CACHE_MNG_ADDR + \
+#define DRAM_KMD_END_ADDR (MMU_CACHE_MNG_ADDR + \
MMU_CACHE_MNG_SIZE)
-#define CPU_PQ_DATA_ADDR (CPU_PQ_PKT_ADDR + CPU_PQ_PKT_SIZE)
-#define DRAM_BASE_ADDR_USER (CPU_PQ_DATA_ADDR + CPU_PQ_DATA_SIZE)
-#if (DRAM_BASE_ADDR_USER != 0x20000000)
-#error "KMD must reserve 512MB"
+#define DRAM_BASE_ADDR_USER 0x20000000
+
+#if (DRAM_KMD_END_ADDR > DRAM_BASE_ADDR_USER)
+#error "KMD must reserve no more than 512MB"
#endif
/*
@@ -142,22 +141,12 @@
#define HW_CAP_GOLDEN 0x00000400
#define HW_CAP_TPC 0x00000800
-#define CPU_PKT_SHIFT 5
-#define CPU_PKT_SIZE (1 << CPU_PKT_SHIFT)
-#define CPU_PKT_MASK (~((1 << CPU_PKT_SHIFT) - 1))
-#define CPU_MAX_PKTS_IN_CB 32
-#define CPU_CB_SIZE (CPU_PKT_SIZE * CPU_MAX_PKTS_IN_CB)
-#define CPU_ACCESSIBLE_MEM_SIZE (HL_QUEUE_LENGTH * CPU_CB_SIZE)
-
enum goya_fw_component {
FW_COMP_UBOOT,
FW_COMP_PREBOOT
};
struct goya_device {
- int (*test_cpu_queue)(struct hl_device *hdev);
- int (*armcp_info_get)(struct hl_device *hdev);
-
/* TODO: remove hw_queues_lock after moving to scheduler code */
spinlock_t hw_queues_lock;
@@ -170,13 +159,34 @@ struct goya_device {
u32 hw_cap_initialized;
};
+void goya_get_fixed_properties(struct hl_device *hdev);
+int goya_mmu_init(struct hl_device *hdev);
+void goya_init_dma_qmans(struct hl_device *hdev);
+void goya_init_mme_qmans(struct hl_device *hdev);
+void goya_init_tpc_qmans(struct hl_device *hdev);
+int goya_init_cpu_queues(struct hl_device *hdev);
+void goya_init_security(struct hl_device *hdev);
+int goya_late_init(struct hl_device *hdev);
+void goya_late_fini(struct hl_device *hdev);
+
+void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi);
+void goya_flush_pq_write(struct hl_device *hdev, u64 *pq, u64 exp_val);
+void goya_update_eq_ci(struct hl_device *hdev, u32 val);
+void goya_restore_phase_topology(struct hl_device *hdev);
+int goya_context_switch(struct hl_device *hdev, u32 asid);
+
int goya_debugfs_i2c_read(struct hl_device *hdev, u8 i2c_bus,
u8 i2c_addr, u8 i2c_reg, u32 *val);
int goya_debugfs_i2c_write(struct hl_device *hdev, u8 i2c_bus,
u8 i2c_addr, u8 i2c_reg, u32 val);
+void goya_debugfs_led_set(struct hl_device *hdev, u8 led, u8 state);
+
+int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id);
+int goya_test_queues(struct hl_device *hdev);
int goya_test_cpu_queue(struct hl_device *hdev);
int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
u32 timeout, long *result);
+
long goya_get_temperature(struct hl_device *hdev, int sensor_index, u32 attr);
long goya_get_voltage(struct hl_device *hdev, int sensor_index, u32 attr);
long goya_get_current(struct hl_device *hdev, int sensor_index, u32 attr);
@@ -184,28 +194,35 @@ long goya_get_fan_speed(struct hl_device *hdev, int sensor_index, u32 attr);
long goya_get_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr);
void goya_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr,
long value);
-void goya_debugfs_led_set(struct hl_device *hdev, u8 led, u8 state);
+u64 goya_get_max_power(struct hl_device *hdev);
+void goya_set_max_power(struct hl_device *hdev, u64 value);
+
void goya_set_pll_profile(struct hl_device *hdev, enum hl_pll_frequency freq);
void goya_add_device_attr(struct hl_device *hdev,
struct attribute_group *dev_attr_grp);
-void goya_init_security(struct hl_device *hdev);
-u64 goya_get_max_power(struct hl_device *hdev);
-void goya_set_max_power(struct hl_device *hdev, u64 value);
+int goya_armcp_info_get(struct hl_device *hdev);
+int goya_debug_coresight(struct hl_device *hdev, void *data);
+
+void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
+int goya_mmu_clear_pgt_range(struct hl_device *hdev);
+int goya_mmu_set_dram_default_page(struct hl_device *hdev);
-int goya_send_pci_access_msg(struct hl_device *hdev, u32 opcode);
-void goya_late_fini(struct hl_device *hdev);
int goya_suspend(struct hl_device *hdev);
int goya_resume(struct hl_device *hdev);
-void goya_flush_pq_write(struct hl_device *hdev, u64 *pq, u64 exp_val);
+
void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry);
void *goya_get_events_stat(struct hl_device *hdev, u32 *size);
+
void goya_add_end_of_cb_packets(u64 kernel_address, u32 len, u64 cq_addr,
u32 cq_val, u32 msix_vec);
int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser);
void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id,
- dma_addr_t *dma_handle, u16 *queue_len);
+ dma_addr_t *dma_handle, u16 *queue_len);
u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt);
-int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id);
int goya_send_heartbeat(struct hl_device *hdev);
+void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
+ dma_addr_t *dma_handle);
+void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
+ void *vaddr);
#endif /* GOYAP_H_ */
diff --git a/drivers/misc/habanalabs/goya/goya_coresight.c b/drivers/misc/habanalabs/goya/goya_coresight.c
new file mode 100644
index 000000000000..1ac951f52d1e
--- /dev/null
+++ b/drivers/misc/habanalabs/goya/goya_coresight.c
@@ -0,0 +1,628 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2019 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#include "goyaP.h"
+#include "include/goya/goya_coresight.h"
+#include "include/goya/asic_reg/goya_regs.h"
+
+#include <uapi/misc/habanalabs.h>
+
+#include <linux/coresight.h>
+
+#define GOYA_PLDM_CORESIGHT_TIMEOUT_USEC (CORESIGHT_TIMEOUT_USEC * 100)
+
+static u64 debug_stm_regs[GOYA_STM_LAST + 1] = {
+ [GOYA_STM_CPU] = mmCPU_STM_BASE,
+ [GOYA_STM_DMA_CH_0_CS] = mmDMA_CH_0_CS_STM_BASE,
+ [GOYA_STM_DMA_CH_1_CS] = mmDMA_CH_1_CS_STM_BASE,
+ [GOYA_STM_DMA_CH_2_CS] = mmDMA_CH_2_CS_STM_BASE,
+ [GOYA_STM_DMA_CH_3_CS] = mmDMA_CH_3_CS_STM_BASE,
+ [GOYA_STM_DMA_CH_4_CS] = mmDMA_CH_4_CS_STM_BASE,
+ [GOYA_STM_DMA_MACRO_CS] = mmDMA_MACRO_CS_STM_BASE,
+ [GOYA_STM_MME1_SBA] = mmMME1_SBA_STM_BASE,
+ [GOYA_STM_MME3_SBB] = mmMME3_SBB_STM_BASE,
+ [GOYA_STM_MME4_WACS2] = mmMME4_WACS2_STM_BASE,
+ [GOYA_STM_MME4_WACS] = mmMME4_WACS_STM_BASE,
+ [GOYA_STM_MMU_CS] = mmMMU_CS_STM_BASE,
+ [GOYA_STM_PCIE] = mmPCIE_STM_BASE,
+ [GOYA_STM_PSOC] = mmPSOC_STM_BASE,
+ [GOYA_STM_TPC0_EML] = mmTPC0_EML_STM_BASE,
+ [GOYA_STM_TPC1_EML] = mmTPC1_EML_STM_BASE,
+ [GOYA_STM_TPC2_EML] = mmTPC2_EML_STM_BASE,
+ [GOYA_STM_TPC3_EML] = mmTPC3_EML_STM_BASE,
+ [GOYA_STM_TPC4_EML] = mmTPC4_EML_STM_BASE,
+ [GOYA_STM_TPC5_EML] = mmTPC5_EML_STM_BASE,
+ [GOYA_STM_TPC6_EML] = mmTPC6_EML_STM_BASE,
+ [GOYA_STM_TPC7_EML] = mmTPC7_EML_STM_BASE
+};
+
+static u64 debug_etf_regs[GOYA_ETF_LAST + 1] = {
+ [GOYA_ETF_CPU_0] = mmCPU_ETF_0_BASE,
+ [GOYA_ETF_CPU_1] = mmCPU_ETF_1_BASE,
+ [GOYA_ETF_CPU_TRACE] = mmCPU_ETF_TRACE_BASE,
+ [GOYA_ETF_DMA_CH_0_CS] = mmDMA_CH_0_CS_ETF_BASE,
+ [GOYA_ETF_DMA_CH_1_CS] = mmDMA_CH_1_CS_ETF_BASE,
+ [GOYA_ETF_DMA_CH_2_CS] = mmDMA_CH_2_CS_ETF_BASE,
+ [GOYA_ETF_DMA_CH_3_CS] = mmDMA_CH_3_CS_ETF_BASE,
+ [GOYA_ETF_DMA_CH_4_CS] = mmDMA_CH_4_CS_ETF_BASE,
+ [GOYA_ETF_DMA_MACRO_CS] = mmDMA_MACRO_CS_ETF_BASE,
+ [GOYA_ETF_MME1_SBA] = mmMME1_SBA_ETF_BASE,
+ [GOYA_ETF_MME3_SBB] = mmMME3_SBB_ETF_BASE,
+ [GOYA_ETF_MME4_WACS2] = mmMME4_WACS2_ETF_BASE,
+ [GOYA_ETF_MME4_WACS] = mmMME4_WACS_ETF_BASE,
+ [GOYA_ETF_MMU_CS] = mmMMU_CS_ETF_BASE,
+ [GOYA_ETF_PCIE] = mmPCIE_ETF_BASE,
+ [GOYA_ETF_PSOC] = mmPSOC_ETF_BASE,
+ [GOYA_ETF_TPC0_EML] = mmTPC0_EML_ETF_BASE,
+ [GOYA_ETF_TPC1_EML] = mmTPC1_EML_ETF_BASE,
+ [GOYA_ETF_TPC2_EML] = mmTPC2_EML_ETF_BASE,
+ [GOYA_ETF_TPC3_EML] = mmTPC3_EML_ETF_BASE,
+ [GOYA_ETF_TPC4_EML] = mmTPC4_EML_ETF_BASE,
+ [GOYA_ETF_TPC5_EML] = mmTPC5_EML_ETF_BASE,
+ [GOYA_ETF_TPC6_EML] = mmTPC6_EML_ETF_BASE,
+ [GOYA_ETF_TPC7_EML] = mmTPC7_EML_ETF_BASE
+};
+
+static u64 debug_funnel_regs[GOYA_FUNNEL_LAST + 1] = {
+ [GOYA_FUNNEL_CPU] = mmCPU_FUNNEL_BASE,
+ [GOYA_FUNNEL_DMA_CH_6_1] = mmDMA_CH_FUNNEL_6_1_BASE,
+ [GOYA_FUNNEL_DMA_MACRO_3_1] = mmDMA_MACRO_FUNNEL_3_1_BASE,
+ [GOYA_FUNNEL_MME0_RTR] = mmMME0_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_MME1_RTR] = mmMME1_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_MME2_RTR] = mmMME2_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_MME3_RTR] = mmMME3_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_MME4_RTR] = mmMME4_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_MME5_RTR] = mmMME5_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_PCIE] = mmPCIE_FUNNEL_BASE,
+ [GOYA_FUNNEL_PSOC] = mmPSOC_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC0_EML] = mmTPC0_EML_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC1_EML] = mmTPC1_EML_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC1_RTR] = mmTPC1_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC2_EML] = mmTPC2_EML_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC2_RTR] = mmTPC2_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC3_EML] = mmTPC3_EML_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC3_RTR] = mmTPC3_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC4_EML] = mmTPC4_EML_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC4_RTR] = mmTPC4_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC5_EML] = mmTPC5_EML_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC5_RTR] = mmTPC5_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC6_EML] = mmTPC6_EML_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC6_RTR] = mmTPC6_RTR_FUNNEL_BASE,
+ [GOYA_FUNNEL_TPC7_EML] = mmTPC7_EML_FUNNEL_BASE
+};
+
+static u64 debug_bmon_regs[GOYA_BMON_LAST + 1] = {
+ [GOYA_BMON_CPU_RD] = mmCPU_RD_BMON_BASE,
+ [GOYA_BMON_CPU_WR] = mmCPU_WR_BMON_BASE,
+ [GOYA_BMON_DMA_CH_0_0] = mmDMA_CH_0_BMON_0_BASE,
+ [GOYA_BMON_DMA_CH_0_1] = mmDMA_CH_0_BMON_1_BASE,
+ [GOYA_BMON_DMA_CH_1_0] = mmDMA_CH_1_BMON_0_BASE,
+ [GOYA_BMON_DMA_CH_1_1] = mmDMA_CH_1_BMON_1_BASE,
+ [GOYA_BMON_DMA_CH_2_0] = mmDMA_CH_2_BMON_0_BASE,
+ [GOYA_BMON_DMA_CH_2_1] = mmDMA_CH_2_BMON_1_BASE,
+ [GOYA_BMON_DMA_CH_3_0] = mmDMA_CH_3_BMON_0_BASE,
+ [GOYA_BMON_DMA_CH_3_1] = mmDMA_CH_3_BMON_1_BASE,
+ [GOYA_BMON_DMA_CH_4_0] = mmDMA_CH_4_BMON_0_BASE,
+ [GOYA_BMON_DMA_CH_4_1] = mmDMA_CH_4_BMON_1_BASE,
+ [GOYA_BMON_DMA_MACRO_0] = mmDMA_MACRO_BMON_0_BASE,
+ [GOYA_BMON_DMA_MACRO_1] = mmDMA_MACRO_BMON_1_BASE,
+ [GOYA_BMON_DMA_MACRO_2] = mmDMA_MACRO_BMON_2_BASE,
+ [GOYA_BMON_DMA_MACRO_3] = mmDMA_MACRO_BMON_3_BASE,
+ [GOYA_BMON_DMA_MACRO_4] = mmDMA_MACRO_BMON_4_BASE,
+ [GOYA_BMON_DMA_MACRO_5] = mmDMA_MACRO_BMON_5_BASE,
+ [GOYA_BMON_DMA_MACRO_6] = mmDMA_MACRO_BMON_6_BASE,
+ [GOYA_BMON_DMA_MACRO_7] = mmDMA_MACRO_BMON_7_BASE,
+ [GOYA_BMON_MME1_SBA_0] = mmMME1_SBA_BMON0_BASE,
+ [GOYA_BMON_MME1_SBA_1] = mmMME1_SBA_BMON1_BASE,
+ [GOYA_BMON_MME3_SBB_0] = mmMME3_SBB_BMON0_BASE,
+ [GOYA_BMON_MME3_SBB_1] = mmMME3_SBB_BMON1_BASE,
+ [GOYA_BMON_MME4_WACS2_0] = mmMME4_WACS2_BMON0_BASE,
+ [GOYA_BMON_MME4_WACS2_1] = mmMME4_WACS2_BMON1_BASE,
+ [GOYA_BMON_MME4_WACS2_2] = mmMME4_WACS2_BMON2_BASE,
+ [GOYA_BMON_MME4_WACS_0] = mmMME4_WACS_BMON0_BASE,
+ [GOYA_BMON_MME4_WACS_1] = mmMME4_WACS_BMON1_BASE,
+ [GOYA_BMON_MME4_WACS_2] = mmMME4_WACS_BMON2_BASE,
+ [GOYA_BMON_MME4_WACS_3] = mmMME4_WACS_BMON3_BASE,
+ [GOYA_BMON_MME4_WACS_4] = mmMME4_WACS_BMON4_BASE,
+ [GOYA_BMON_MME4_WACS_5] = mmMME4_WACS_BMON5_BASE,
+ [GOYA_BMON_MME4_WACS_6] = mmMME4_WACS_BMON6_BASE,
+ [GOYA_BMON_MMU_0] = mmMMU_BMON_0_BASE,
+ [GOYA_BMON_MMU_1] = mmMMU_BMON_1_BASE,
+ [GOYA_BMON_PCIE_MSTR_RD] = mmPCIE_BMON_MSTR_RD_BASE,
+ [GOYA_BMON_PCIE_MSTR_WR] = mmPCIE_BMON_MSTR_WR_BASE,
+ [GOYA_BMON_PCIE_SLV_RD] = mmPCIE_BMON_SLV_RD_BASE,
+ [GOYA_BMON_PCIE_SLV_WR] = mmPCIE_BMON_SLV_WR_BASE,
+ [GOYA_BMON_TPC0_EML_0] = mmTPC0_EML_BUSMON_0_BASE,
+ [GOYA_BMON_TPC0_EML_1] = mmTPC0_EML_BUSMON_1_BASE,
+ [GOYA_BMON_TPC0_EML_2] = mmTPC0_EML_BUSMON_2_BASE,
+ [GOYA_BMON_TPC0_EML_3] = mmTPC0_EML_BUSMON_3_BASE,
+ [GOYA_BMON_TPC1_EML_0] = mmTPC1_EML_BUSMON_0_BASE,
+ [GOYA_BMON_TPC1_EML_1] = mmTPC1_EML_BUSMON_1_BASE,
+ [GOYA_BMON_TPC1_EML_2] = mmTPC1_EML_BUSMON_2_BASE,
+ [GOYA_BMON_TPC1_EML_3] = mmTPC1_EML_BUSMON_3_BASE,
+ [GOYA_BMON_TPC2_EML_0] = mmTPC2_EML_BUSMON_0_BASE,
+ [GOYA_BMON_TPC2_EML_1] = mmTPC2_EML_BUSMON_1_BASE,
+ [GOYA_BMON_TPC2_EML_2] = mmTPC2_EML_BUSMON_2_BASE,
+ [GOYA_BMON_TPC2_EML_3] = mmTPC2_EML_BUSMON_3_BASE,
+ [GOYA_BMON_TPC3_EML_0] = mmTPC3_EML_BUSMON_0_BASE,
+ [GOYA_BMON_TPC3_EML_1] = mmTPC3_EML_BUSMON_1_BASE,
+ [GOYA_BMON_TPC3_EML_2] = mmTPC3_EML_BUSMON_2_BASE,
+ [GOYA_BMON_TPC3_EML_3] = mmTPC3_EML_BUSMON_3_BASE,
+ [GOYA_BMON_TPC4_EML_0] = mmTPC4_EML_BUSMON_0_BASE,
+ [GOYA_BMON_TPC4_EML_1] = mmTPC4_EML_BUSMON_1_BASE,
+ [GOYA_BMON_TPC4_EML_2] = mmTPC4_EML_BUSMON_2_BASE,
+ [GOYA_BMON_TPC4_EML_3] = mmTPC4_EML_BUSMON_3_BASE,
+ [GOYA_BMON_TPC5_EML_0] = mmTPC5_EML_BUSMON_0_BASE,
+ [GOYA_BMON_TPC5_EML_1] = mmTPC5_EML_BUSMON_1_BASE,
+ [GOYA_BMON_TPC5_EML_2] = mmTPC5_EML_BUSMON_2_BASE,
+ [GOYA_BMON_TPC5_EML_3] = mmTPC5_EML_BUSMON_3_BASE,
+ [GOYA_BMON_TPC6_EML_0] = mmTPC6_EML_BUSMON_0_BASE,
+ [GOYA_BMON_TPC6_EML_1] = mmTPC6_EML_BUSMON_1_BASE,
+ [GOYA_BMON_TPC6_EML_2] = mmTPC6_EML_BUSMON_2_BASE,
+ [GOYA_BMON_TPC6_EML_3] = mmTPC6_EML_BUSMON_3_BASE,
+ [GOYA_BMON_TPC7_EML_0] = mmTPC7_EML_BUSMON_0_BASE,
+ [GOYA_BMON_TPC7_EML_1] = mmTPC7_EML_BUSMON_1_BASE,
+ [GOYA_BMON_TPC7_EML_2] = mmTPC7_EML_BUSMON_2_BASE,
+ [GOYA_BMON_TPC7_EML_3] = mmTPC7_EML_BUSMON_3_BASE
+};
+
+static u64 debug_spmu_regs[GOYA_SPMU_LAST + 1] = {
+ [GOYA_SPMU_DMA_CH_0_CS] = mmDMA_CH_0_CS_SPMU_BASE,
+ [GOYA_SPMU_DMA_CH_1_CS] = mmDMA_CH_1_CS_SPMU_BASE,
+ [GOYA_SPMU_DMA_CH_2_CS] = mmDMA_CH_2_CS_SPMU_BASE,
+ [GOYA_SPMU_DMA_CH_3_CS] = mmDMA_CH_3_CS_SPMU_BASE,
+ [GOYA_SPMU_DMA_CH_4_CS] = mmDMA_CH_4_CS_SPMU_BASE,
+ [GOYA_SPMU_DMA_MACRO_CS] = mmDMA_MACRO_CS_SPMU_BASE,
+ [GOYA_SPMU_MME1_SBA] = mmMME1_SBA_SPMU_BASE,
+ [GOYA_SPMU_MME3_SBB] = mmMME3_SBB_SPMU_BASE,
+ [GOYA_SPMU_MME4_WACS2] = mmMME4_WACS2_SPMU_BASE,
+ [GOYA_SPMU_MME4_WACS] = mmMME4_WACS_SPMU_BASE,
+ [GOYA_SPMU_MMU_CS] = mmMMU_CS_SPMU_BASE,
+ [GOYA_SPMU_PCIE] = mmPCIE_SPMU_BASE,
+ [GOYA_SPMU_TPC0_EML] = mmTPC0_EML_SPMU_BASE,
+ [GOYA_SPMU_TPC1_EML] = mmTPC1_EML_SPMU_BASE,
+ [GOYA_SPMU_TPC2_EML] = mmTPC2_EML_SPMU_BASE,
+ [GOYA_SPMU_TPC3_EML] = mmTPC3_EML_SPMU_BASE,
+ [GOYA_SPMU_TPC4_EML] = mmTPC4_EML_SPMU_BASE,
+ [GOYA_SPMU_TPC5_EML] = mmTPC5_EML_SPMU_BASE,
+ [GOYA_SPMU_TPC6_EML] = mmTPC6_EML_SPMU_BASE,
+ [GOYA_SPMU_TPC7_EML] = mmTPC7_EML_SPMU_BASE
+};
+
+static int goya_coresight_timeout(struct hl_device *hdev, u64 addr,
+ int position, bool up)
+{
+ int rc;
+ u32 val, timeout_usec;
+
+ if (hdev->pldm)
+ timeout_usec = GOYA_PLDM_CORESIGHT_TIMEOUT_USEC;
+ else
+ timeout_usec = CORESIGHT_TIMEOUT_USEC;
+
+ rc = hl_poll_timeout(
+ hdev,
+ addr,
+ val,
+ up ? val & BIT(position) : !(val & BIT(position)),
+ 1000,
+ timeout_usec);
+
+ if (rc) {
+ dev_err(hdev->dev,
+ "Timeout while waiting for coresight, addr: 0x%llx, position: %d, up: %d\n",
+ addr, position, up);
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+static int goya_config_stm(struct hl_device *hdev,
+ struct hl_debug_params *params)
+{
+ struct hl_debug_params_stm *input;
+ u64 base_reg = debug_stm_regs[params->reg_idx] - CFG_BASE;
+ int rc;
+
+ WREG32(base_reg + 0xFB0, CORESIGHT_UNLOCK);
+
+ if (params->enable) {
+ input = params->input;
+
+ if (!input)
+ return -EINVAL;
+
+ WREG32(base_reg + 0xE80, 0x80004);
+ WREG32(base_reg + 0xD64, 7);
+ WREG32(base_reg + 0xD60, 0);
+ WREG32(base_reg + 0xD00, lower_32_bits(input->he_mask));
+ WREG32(base_reg + 0xD20, lower_32_bits(input->sp_mask));
+ WREG32(base_reg + 0xD60, 1);
+ WREG32(base_reg + 0xD00, upper_32_bits(input->he_mask));
+ WREG32(base_reg + 0xD20, upper_32_bits(input->sp_mask));
+ WREG32(base_reg + 0xE70, 0x10);
+ WREG32(base_reg + 0xE60, 0);
+ WREG32(base_reg + 0xE64, 0x420000);
+ WREG32(base_reg + 0xE00, 0xFFFFFFFF);
+ WREG32(base_reg + 0xE20, 0xFFFFFFFF);
+ WREG32(base_reg + 0xEF4, input->id);
+ WREG32(base_reg + 0xDF4, 0x80);
+ WREG32(base_reg + 0xE8C, input->frequency);
+ WREG32(base_reg + 0xE90, 0x7FF);
+ WREG32(base_reg + 0xE80, 0x7 | (input->id << 16));
+ } else {
+ WREG32(base_reg + 0xE80, 4);
+ WREG32(base_reg + 0xD64, 0);
+ WREG32(base_reg + 0xD60, 1);
+ WREG32(base_reg + 0xD00, 0);
+ WREG32(base_reg + 0xD20, 0);
+ WREG32(base_reg + 0xD60, 0);
+ WREG32(base_reg + 0xE20, 0);
+ WREG32(base_reg + 0xE00, 0);
+ WREG32(base_reg + 0xDF4, 0x80);
+ WREG32(base_reg + 0xE70, 0);
+ WREG32(base_reg + 0xE60, 0);
+ WREG32(base_reg + 0xE64, 0);
+ WREG32(base_reg + 0xE8C, 0);
+
+ rc = goya_coresight_timeout(hdev, base_reg + 0xE80, 23, false);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to disable STM on timeout, error %d\n",
+ rc);
+ return rc;
+ }
+
+ WREG32(base_reg + 0xE80, 4);
+ }
+
+ return 0;
+}
+
+static int goya_config_etf(struct hl_device *hdev,
+ struct hl_debug_params *params)
+{
+ struct hl_debug_params_etf *input;
+ u64 base_reg = debug_etf_regs[params->reg_idx] - CFG_BASE;
+ u32 val;
+ int rc;
+
+ WREG32(base_reg + 0xFB0, CORESIGHT_UNLOCK);
+
+ val = RREG32(base_reg + 0x304);
+ val |= 0x1000;
+ WREG32(base_reg + 0x304, val);
+ val |= 0x40;
+ WREG32(base_reg + 0x304, val);
+
+ rc = goya_coresight_timeout(hdev, base_reg + 0x304, 6, false);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to %s ETF on timeout, error %d\n",
+ params->enable ? "enable" : "disable", rc);
+ return rc;
+ }
+
+ rc = goya_coresight_timeout(hdev, base_reg + 0xC, 2, true);
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to %s ETF on timeout, error %d\n",
+ params->enable ? "enable" : "disable", rc);
+ return rc;
+ }
+
+ WREG32(base_reg + 0x20, 0);
+
+ if (params->enable) {
+ input = params->input;
+
+ if (!input)
+ return -EINVAL;
+
+ WREG32(base_reg + 0x34, 0x3FFC);
+ WREG32(base_reg + 0x28, input->sink_mode);
+ WREG32(base_reg + 0x304, 0x4001);
+ WREG32(base_reg + 0x308, 0xA);
+ WREG32(base_reg + 0x20, 1);
+ } else {
+ WREG32(base_reg + 0x34, 0);
+ WREG32(base_reg + 0x28, 0);
+ WREG32(base_reg + 0x304, 0);
+ }
+
+ return 0;
+}
+
+static int goya_etr_validate_address(struct hl_device *hdev, u64 addr,
+ u32 size)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u64 range_start, range_end;
+
+ if (hdev->mmu_enable) {
+ range_start = prop->va_space_dram_start_address;
+ range_end = prop->va_space_dram_end_address;
+ } else {
+ range_start = prop->dram_user_base_address;
+ range_end = prop->dram_end_address;
+ }
+
+ return hl_mem_area_inside_range(addr, size, range_start, range_end);
+}
+
+static int goya_config_etr(struct hl_device *hdev,
+ struct hl_debug_params *params)
+{
+ struct hl_debug_params_etr *input;
+ u64 base_reg = mmPSOC_ETR_BASE - CFG_BASE;
+ u32 val;
+ int rc;
+
+ WREG32(base_reg + 0xFB0, CORESIGHT_UNLOCK);
+
+ val = RREG32(base_reg + 0x304);
+ val |= 0x1000;
+ WREG32(base_reg + 0x304, val);
+ val |= 0x40;
+ WREG32(base_reg + 0x304, val);
+
+ rc = goya_coresight_timeout(hdev, base_reg + 0x304, 6, false);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to %s ETR on timeout, error %d\n",
+ params->enable ? "enable" : "disable", rc);
+ return rc;
+ }
+
+ rc = goya_coresight_timeout(hdev, base_reg + 0xC, 2, true);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to %s ETR on timeout, error %d\n",
+ params->enable ? "enable" : "disable", rc);
+ return rc;
+ }
+
+ WREG32(base_reg + 0x20, 0);
+
+ if (params->enable) {
+ input = params->input;
+
+ if (!input)
+ return -EINVAL;
+
+ if (input->buffer_size == 0) {
+ dev_err(hdev->dev,
+ "ETR buffer size should be bigger than 0\n");
+ return -EINVAL;
+ }
+
+ if (!goya_etr_validate_address(hdev,
+ input->buffer_address, input->buffer_size)) {
+ dev_err(hdev->dev, "buffer address is not valid\n");
+ return -EINVAL;
+ }
+
+ WREG32(base_reg + 0x34, 0x3FFC);
+ WREG32(base_reg + 0x4, input->buffer_size);
+ WREG32(base_reg + 0x28, input->sink_mode);
+ WREG32(base_reg + 0x110, 0x700);
+ WREG32(base_reg + 0x118,
+ lower_32_bits(input->buffer_address));
+ WREG32(base_reg + 0x11C,
+ upper_32_bits(input->buffer_address));
+ WREG32(base_reg + 0x304, 3);
+ WREG32(base_reg + 0x308, 0xA);
+ WREG32(base_reg + 0x20, 1);
+ } else {
+ WREG32(base_reg + 0x34, 0);
+ WREG32(base_reg + 0x4, 0x400);
+ WREG32(base_reg + 0x118, 0);
+ WREG32(base_reg + 0x11C, 0);
+ WREG32(base_reg + 0x308, 0);
+ WREG32(base_reg + 0x28, 0);
+ WREG32(base_reg + 0x304, 0);
+
+ if (params->output_size >= sizeof(u32))
+ *(u32 *) params->output = RREG32(base_reg + 0x18);
+ }
+
+ return 0;
+}
+
+static int goya_config_funnel(struct hl_device *hdev,
+ struct hl_debug_params *params)
+{
+ WREG32(debug_funnel_regs[params->reg_idx] - CFG_BASE + 0xFB0,
+ CORESIGHT_UNLOCK);
+
+ WREG32(debug_funnel_regs[params->reg_idx] - CFG_BASE,
+ params->enable ? 0x33F : 0);
+
+ return 0;
+}
+
+static int goya_config_bmon(struct hl_device *hdev,
+ struct hl_debug_params *params)
+{
+ struct hl_debug_params_bmon *input;
+ u64 base_reg = debug_bmon_regs[params->reg_idx] - CFG_BASE;
+ u32 pcie_base = 0;
+
+ WREG32(base_reg + 0x104, 1);
+
+ if (params->enable) {
+ input = params->input;
+
+ if (!input)
+ return -EINVAL;
+
+ WREG32(base_reg + 0x200, lower_32_bits(input->start_addr0));
+ WREG32(base_reg + 0x204, upper_32_bits(input->start_addr0));
+ WREG32(base_reg + 0x208, lower_32_bits(input->addr_mask0));
+ WREG32(base_reg + 0x20C, upper_32_bits(input->addr_mask0));
+ WREG32(base_reg + 0x240, lower_32_bits(input->start_addr1));
+ WREG32(base_reg + 0x244, upper_32_bits(input->start_addr1));
+ WREG32(base_reg + 0x248, lower_32_bits(input->addr_mask1));
+ WREG32(base_reg + 0x24C, upper_32_bits(input->addr_mask1));
+ WREG32(base_reg + 0x224, 0);
+ WREG32(base_reg + 0x234, 0);
+ WREG32(base_reg + 0x30C, input->bw_win);
+ WREG32(base_reg + 0x308, input->win_capture);
+
+ /* PCIE IF BMON bug WA */
+ if (params->reg_idx != GOYA_BMON_PCIE_MSTR_RD &&
+ params->reg_idx != GOYA_BMON_PCIE_MSTR_WR &&
+ params->reg_idx != GOYA_BMON_PCIE_SLV_RD &&
+ params->reg_idx != GOYA_BMON_PCIE_SLV_WR)
+ pcie_base = 0xA000000;
+
+ WREG32(base_reg + 0x700, pcie_base | 0xB00 | (input->id << 12));
+ WREG32(base_reg + 0x708, pcie_base | 0xA00 | (input->id << 12));
+ WREG32(base_reg + 0x70C, pcie_base | 0xC00 | (input->id << 12));
+
+ WREG32(base_reg + 0x100, 0x11);
+ WREG32(base_reg + 0x304, 0x1);
+ } else {
+ WREG32(base_reg + 0x200, 0);
+ WREG32(base_reg + 0x204, 0);
+ WREG32(base_reg + 0x208, 0xFFFFFFFF);
+ WREG32(base_reg + 0x20C, 0xFFFFFFFF);
+ WREG32(base_reg + 0x240, 0);
+ WREG32(base_reg + 0x244, 0);
+ WREG32(base_reg + 0x248, 0xFFFFFFFF);
+ WREG32(base_reg + 0x24C, 0xFFFFFFFF);
+ WREG32(base_reg + 0x224, 0xFFFFFFFF);
+ WREG32(base_reg + 0x234, 0x1070F);
+ WREG32(base_reg + 0x30C, 0);
+ WREG32(base_reg + 0x308, 0xFFFF);
+ WREG32(base_reg + 0x700, 0xA000B00);
+ WREG32(base_reg + 0x708, 0xA000A00);
+ WREG32(base_reg + 0x70C, 0xA000C00);
+ WREG32(base_reg + 0x100, 1);
+ WREG32(base_reg + 0x304, 0);
+ WREG32(base_reg + 0x104, 0);
+ }
+
+ return 0;
+}
+
+static int goya_config_spmu(struct hl_device *hdev,
+ struct hl_debug_params *params)
+{
+ u64 base_reg = debug_spmu_regs[params->reg_idx] - CFG_BASE;
+ struct hl_debug_params_spmu *input = params->input;
+ u64 *output;
+ u32 output_arr_len;
+ u32 events_num;
+ u32 overflow_idx;
+ u32 cycle_cnt_idx;
+ int i;
+
+ if (params->enable) {
+ input = params->input;
+
+ if (!input)
+ return -EINVAL;
+
+ if (input->event_types_num < 3) {
+ dev_err(hdev->dev,
+ "not enough values for SPMU enable\n");
+ return -EINVAL;
+ }
+
+ WREG32(base_reg + 0xE04, 0x41013046);
+ WREG32(base_reg + 0xE04, 0x41013040);
+
+ for (i = 0 ; i < input->event_types_num ; i++)
+ WREG32(base_reg + 0x400 + i * 4, input->event_types[i]);
+
+ WREG32(base_reg + 0xE04, 0x41013041);
+ WREG32(base_reg + 0xC00, 0x8000003F);
+ } else {
+ output = params->output;
+ output_arr_len = params->output_size / 8;
+ events_num = output_arr_len - 2;
+ overflow_idx = output_arr_len - 2;
+ cycle_cnt_idx = output_arr_len - 1;
+
+ if (!output)
+ return -EINVAL;
+
+ if (output_arr_len < 3) {
+ dev_err(hdev->dev,
+ "not enough values for SPMU disable\n");
+ return -EINVAL;
+ }
+
+ WREG32(base_reg + 0xE04, 0x41013040);
+
+ for (i = 0 ; i < events_num ; i++)
+ output[i] = RREG32(base_reg + i * 8);
+
+ output[overflow_idx] = RREG32(base_reg + 0xCC0);
+
+ output[cycle_cnt_idx] = RREG32(base_reg + 0xFC);
+ output[cycle_cnt_idx] <<= 32;
+ output[cycle_cnt_idx] |= RREG32(base_reg + 0xF8);
+
+ WREG32(base_reg + 0xCC0, 0);
+ }
+
+ return 0;
+}
+
+static int goya_config_timestamp(struct hl_device *hdev,
+ struct hl_debug_params *params)
+{
+ WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
+ if (params->enable) {
+ WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0);
+ WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0);
+ WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1);
+ }
+
+ return 0;
+}
+
+int goya_debug_coresight(struct hl_device *hdev, void *data)
+{
+ struct hl_debug_params *params = data;
+ u32 val;
+ int rc;
+
+ switch (params->op) {
+ case HL_DEBUG_OP_STM:
+ rc = goya_config_stm(hdev, params);
+ break;
+ case HL_DEBUG_OP_ETF:
+ rc = goya_config_etf(hdev, params);
+ break;
+ case HL_DEBUG_OP_ETR:
+ rc = goya_config_etr(hdev, params);
+ break;
+ case HL_DEBUG_OP_FUNNEL:
+ rc = goya_config_funnel(hdev, params);
+ break;
+ case HL_DEBUG_OP_BMON:
+ rc = goya_config_bmon(hdev, params);
+ break;
+ case HL_DEBUG_OP_SPMU:
+ rc = goya_config_spmu(hdev, params);
+ break;
+ case HL_DEBUG_OP_TIMESTAMP:
+ rc = goya_config_timestamp(hdev, params);
+ break;
+
+ default:
+ dev_err(hdev->dev, "Unknown coresight id %d\n", params->op);
+ return -EINVAL;
+ }
+
+ /* Perform read from the device to flush all configuration */
+ val = RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
+
+ return rc;
+}
diff --git a/drivers/misc/habanalabs/goya/goya_security.c b/drivers/misc/habanalabs/goya/goya_security.c
index 575003238401..d95d1b2f860d 100644
--- a/drivers/misc/habanalabs/goya/goya_security.c
+++ b/drivers/misc/habanalabs/goya/goya_security.c
@@ -6,6 +6,7 @@
*/
#include "goyaP.h"
+#include "include/goya/asic_reg/goya_regs.h"
/*
* goya_set_block_as_protected - set the given block as protected
@@ -2159,6 +2160,8 @@ static void goya_init_protection_bits(struct hl_device *hdev)
* Bits 7-11 represents the word offset inside the 128 bytes.
* Bits 2-6 represents the bit location inside the word.
*/
+ u32 pb_addr, mask;
+ u8 word_offset;
goya_pb_set_block(hdev, mmPCI_NRTR_BASE);
goya_pb_set_block(hdev, mmPCI_RD_REGULATOR_BASE);
@@ -2237,6 +2240,14 @@ static void goya_init_protection_bits(struct hl_device *hdev)
goya_pb_set_block(hdev, mmPCIE_AUX_BASE);
goya_pb_set_block(hdev, mmPCIE_DB_RSV_BASE);
goya_pb_set_block(hdev, mmPCIE_PHY_BASE);
+ goya_pb_set_block(hdev, mmTPC0_NRTR_BASE);
+ goya_pb_set_block(hdev, mmTPC_PLL_BASE);
+
+ pb_addr = (mmTPC_PLL_CLK_RLX_0 & ~0xFFF) + PROT_BITS_OFFS;
+ word_offset = ((mmTPC_PLL_CLK_RLX_0 & PROT_BITS_OFFS) >> 7) << 2;
+ mask = 1 << ((mmTPC_PLL_CLK_RLX_0 & 0x7C) >> 2);
+
+ WREG32(pb_addr + word_offset, mask);
goya_init_mme_protection_bits(hdev);
@@ -2294,8 +2305,8 @@ void goya_init_security(struct hl_device *hdev)
u32 lbw_rng10_base = 0xFCC60000 & DMA_MACRO_LBW_RANGE_BASE_R_MASK;
u32 lbw_rng10_mask = 0xFFFE0000 & DMA_MACRO_LBW_RANGE_BASE_R_MASK;
- u32 lbw_rng11_base = 0xFCE00000 & DMA_MACRO_LBW_RANGE_BASE_R_MASK;
- u32 lbw_rng11_mask = 0xFFFFC000 & DMA_MACRO_LBW_RANGE_BASE_R_MASK;
+ u32 lbw_rng11_base = 0xFCE02000 & DMA_MACRO_LBW_RANGE_BASE_R_MASK;
+ u32 lbw_rng11_mask = 0xFFFFE000 & DMA_MACRO_LBW_RANGE_BASE_R_MASK;
u32 lbw_rng12_base = 0xFE484000 & DMA_MACRO_LBW_RANGE_BASE_R_MASK;
u32 lbw_rng12_mask = 0xFFFFF000 & DMA_MACRO_LBW_RANGE_BASE_R_MASK;
diff --git a/drivers/misc/habanalabs/habanalabs.h b/drivers/misc/habanalabs/habanalabs.h
index a8ee52c880cd..71243b319920 100644
--- a/drivers/misc/habanalabs/habanalabs.h
+++ b/drivers/misc/habanalabs/habanalabs.h
@@ -11,8 +11,6 @@
#include "include/armcp_if.h"
#include "include/qman_if.h"
-#define pr_fmt(fmt) "habanalabs: " fmt
-
#include <linux/cdev.h>
#include <linux/iopoll.h>
#include <linux/irqreturn.h>
@@ -33,6 +31,9 @@
#define HL_PLL_LOW_JOB_FREQ_USEC 5000000 /* 5 s */
+#define HL_ARMCP_INFO_TIMEOUT_USEC 10000000 /* 10s */
+#define HL_ARMCP_EEPROM_TIMEOUT_USEC 10000000 /* 10s */
+
#define HL_MAX_QUEUES 128
#define HL_MAX_JOBS_PER_CS 64
@@ -48,8 +49,9 @@
/**
* struct pgt_info - MMU hop page info.
- * @node: hash linked-list node for the pgts hash of pgts.
- * @addr: physical address of the pgt.
+ * @node: hash linked-list node for the pgts shadow hash of pgts.
+ * @phys_addr: physical address of the pgt.
+ * @shadow_addr: shadow hop in the host.
* @ctx: pointer to the owner ctx.
* @num_of_ptes: indicates how many ptes are used in the pgt.
*
@@ -59,10 +61,11 @@
* page, it is freed with its pgt_info structure.
*/
struct pgt_info {
- struct hlist_node node;
- u64 addr;
- struct hl_ctx *ctx;
- int num_of_ptes;
+ struct hlist_node node;
+ u64 phys_addr;
+ u64 shadow_addr;
+ struct hl_ctx *ctx;
+ int num_of_ptes;
};
struct hl_device;
@@ -132,8 +135,6 @@ enum hl_device_hw_state {
* @dram_user_base_address: DRAM physical start address for user access.
* @dram_size: DRAM total size.
* @dram_pci_bar_size: size of PCI bar towards DRAM.
- * @host_phys_base_address: base physical address of host memory for
- * transactions that the device generates.
* @max_power_default: max power of the device after reset
* @va_space_host_start_address: base address of virtual memory range for
* mapping host memory.
@@ -145,6 +146,8 @@ enum hl_device_hw_state {
* mapping DRAM memory.
* @dram_size_for_default_page_mapping: DRAM size needed to map to avoid page
* fault.
+ * @pcie_dbi_base_address: Base address of the PCIE_DBI block.
+ * @pcie_aux_dbi_reg_addr: Address of the PCIE_AUX DBI register.
* @mmu_pgt_addr: base physical address in DRAM of MMU page tables.
* @mmu_dram_default_page_addr: DRAM default page physical address.
* @mmu_pgt_size: MMU page tables total size.
@@ -179,13 +182,14 @@ struct asic_fixed_properties {
u64 dram_user_base_address;
u64 dram_size;
u64 dram_pci_bar_size;
- u64 host_phys_base_address;
u64 max_power_default;
u64 va_space_host_start_address;
u64 va_space_host_end_address;
u64 va_space_dram_start_address;
u64 va_space_dram_end_address;
u64 dram_size_for_default_page_mapping;
+ u64 pcie_dbi_base_address;
+ u64 pcie_aux_dbi_reg_addr;
u64 mmu_pgt_addr;
u64 mmu_dram_default_page_addr;
u32 mmu_pgt_size;
@@ -314,6 +318,18 @@ struct hl_cs_job;
#define HL_EQ_LENGTH 64
#define HL_EQ_SIZE_IN_BYTES (HL_EQ_LENGTH * HL_EQ_ENTRY_SIZE)
+#define HL_CPU_PKT_SHIFT 5
+#define HL_CPU_PKT_SIZE (1 << HL_CPU_PKT_SHIFT)
+#define HL_CPU_PKT_MASK (~((1 << HL_CPU_PKT_SHIFT) - 1))
+#define HL_CPU_MAX_PKTS_IN_CB 32
+#define HL_CPU_CB_SIZE (HL_CPU_PKT_SIZE * \
+ HL_CPU_MAX_PKTS_IN_CB)
+#define HL_CPU_CB_QUEUE_SIZE (HL_QUEUE_LENGTH * HL_CPU_CB_SIZE)
+
+/* KMD <-> ArmCP shared memory size (EQ + PQ + CPU CB queue) */
+#define HL_CPU_ACCESSIBLE_MEM_SIZE (HL_EQ_SIZE_IN_BYTES + \
+ HL_QUEUE_SIZE_IN_BYTES + \
+ HL_CPU_CB_QUEUE_SIZE)
/**
* struct hl_hw_queue - describes a H/W transport queue.
@@ -381,14 +397,12 @@ struct hl_eq {
/**
* enum hl_asic_type - supported ASIC types.
- * @ASIC_AUTO_DETECT: ASIC type will be automatically set.
- * @ASIC_GOYA: Goya device.
* @ASIC_INVALID: Invalid ASIC type.
+ * @ASIC_GOYA: Goya device.
*/
enum hl_asic_type {
- ASIC_AUTO_DETECT,
- ASIC_GOYA,
- ASIC_INVALID
+ ASIC_INVALID,
+ ASIC_GOYA
};
struct hl_cs_parser;
@@ -436,19 +450,19 @@ enum hl_pll_frequency {
* @cb_mmap: maps a CB.
* @ring_doorbell: increment PI on a given QMAN.
* @flush_pq_write: flush PQ entry write if necessary, WARN if flushing failed.
- * @dma_alloc_coherent: Allocate coherent DMA memory by calling
- * dma_alloc_coherent(). This is ASIC function because its
- * implementation is not trivial when the driver is loaded
- * in simulation mode (not upstreamed).
- * @dma_free_coherent: Free coherent DMA memory by calling dma_free_coherent().
- * This is ASIC function because its implementation is not
- * trivial when the driver is loaded in simulation mode
- * (not upstreamed).
+ * @asic_dma_alloc_coherent: Allocate coherent DMA memory by calling
+ * dma_alloc_coherent(). This is ASIC function because
+ * its implementation is not trivial when the driver
+ * is loaded in simulation mode (not upstreamed).
+ * @asic_dma_free_coherent: Free coherent DMA memory by calling
+ * dma_free_coherent(). This is ASIC function because
+ * its implementation is not trivial when the driver
+ * is loaded in simulation mode (not upstreamed).
* @get_int_queue_base: get the internal queue base address.
* @test_queues: run simple test on all queues for sanity check.
- * @dma_pool_zalloc: small DMA allocation of coherent memory from DMA pool.
- * size of allocation is HL_DMA_POOL_BLK_SIZE.
- * @dma_pool_free: free small DMA allocation from pool.
+ * @asic_dma_pool_zalloc: small DMA allocation of coherent memory from DMA pool.
+ * size of allocation is HL_DMA_POOL_BLK_SIZE.
+ * @asic_dma_pool_free: free small DMA allocation from pool.
* @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool.
* @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool.
* @hl_dma_unmap_sg: DMA unmap scatter-gather list.
@@ -472,8 +486,7 @@ enum hl_pll_frequency {
* @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with
* ASID-VA-size mask.
* @send_heartbeat: send is-alive packet to ArmCP and verify response.
- * @enable_clock_gating: enable clock gating for reducing power consumption.
- * @disable_clock_gating: disable clock for accessing registers on HBW.
+ * @debug_coresight: perform certain actions on Coresight for debugging.
* @is_device_idle: return true if device is idle, false otherwise.
* @soft_reset_late_init: perform certain actions needed after soft reset.
* @hw_queues_lock: acquire H/W queues lock.
@@ -482,6 +495,12 @@ enum hl_pll_frequency {
* @get_eeprom_data: retrieve EEPROM data from F/W.
* @send_cpu_message: send buffer to ArmCP.
* @get_hw_state: retrieve the H/W state
+ * @pci_bars_map: Map PCI BARs.
+ * @set_dram_bar_base: Set DRAM BAR to map specific device address. Returns
+ * old address the bar pointed to or U64_MAX for failure
+ * @init_iatu: Initialize the iATU unit inside the PCI controller.
+ * @rreg: Read a register. Needed for simulator support.
+ * @wreg: Write a register. Needed for simulator support.
*/
struct hl_asic_funcs {
int (*early_init)(struct hl_device *hdev);
@@ -499,27 +518,27 @@ struct hl_asic_funcs {
u64 kaddress, phys_addr_t paddress, u32 size);
void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi);
void (*flush_pq_write)(struct hl_device *hdev, u64 *pq, u64 exp_val);
- void* (*dma_alloc_coherent)(struct hl_device *hdev, size_t size,
+ void* (*asic_dma_alloc_coherent)(struct hl_device *hdev, size_t size,
dma_addr_t *dma_handle, gfp_t flag);
- void (*dma_free_coherent)(struct hl_device *hdev, size_t size,
+ void (*asic_dma_free_coherent)(struct hl_device *hdev, size_t size,
void *cpu_addr, dma_addr_t dma_handle);
void* (*get_int_queue_base)(struct hl_device *hdev, u32 queue_id,
dma_addr_t *dma_handle, u16 *queue_len);
int (*test_queues)(struct hl_device *hdev);
- void* (*dma_pool_zalloc)(struct hl_device *hdev, size_t size,
+ void* (*asic_dma_pool_zalloc)(struct hl_device *hdev, size_t size,
gfp_t mem_flags, dma_addr_t *dma_handle);
- void (*dma_pool_free)(struct hl_device *hdev, void *vaddr,
+ void (*asic_dma_pool_free)(struct hl_device *hdev, void *vaddr,
dma_addr_t dma_addr);
void* (*cpu_accessible_dma_pool_alloc)(struct hl_device *hdev,
size_t size, dma_addr_t *dma_handle);
void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev,
size_t size, void *vaddr);
void (*hl_dma_unmap_sg)(struct hl_device *hdev,
- struct scatterlist *sg, int nents,
+ struct scatterlist *sgl, int nents,
enum dma_data_direction dir);
int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser);
int (*asic_dma_map_sg)(struct hl_device *hdev,
- struct scatterlist *sg, int nents,
+ struct scatterlist *sgl, int nents,
enum dma_data_direction dir);
u32 (*get_dma_desc_list_size)(struct hl_device *hdev,
struct sg_table *sgt);
@@ -543,9 +562,8 @@ struct hl_asic_funcs {
void (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard,
u32 asid, u64 va, u64 size);
int (*send_heartbeat)(struct hl_device *hdev);
- void (*enable_clock_gating)(struct hl_device *hdev);
- void (*disable_clock_gating)(struct hl_device *hdev);
- bool (*is_device_idle)(struct hl_device *hdev);
+ int (*debug_coresight)(struct hl_device *hdev, void *data);
+ bool (*is_device_idle)(struct hl_device *hdev, char *buf, size_t size);
int (*soft_reset_late_init)(struct hl_device *hdev);
void (*hw_queues_lock)(struct hl_device *hdev);
void (*hw_queues_unlock)(struct hl_device *hdev);
@@ -555,6 +573,11 @@ struct hl_asic_funcs {
int (*send_cpu_message)(struct hl_device *hdev, u32 *msg,
u16 len, u32 timeout, long *result);
enum hl_device_hw_state (*get_hw_state)(struct hl_device *hdev);
+ int (*pci_bars_map)(struct hl_device *hdev);
+ u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr);
+ int (*init_iatu)(struct hl_device *hdev);
+ u32 (*rreg)(struct hl_device *hdev, u32 reg);
+ void (*wreg)(struct hl_device *hdev, u32 reg, u32 val);
};
@@ -582,7 +605,8 @@ struct hl_va_range {
* struct hl_ctx - user/kernel context.
* @mem_hash: holds mapping from virtual address to virtual memory area
* descriptor (hl_vm_phys_pg_list or hl_userptr).
- * @mmu_hash: holds a mapping from virtual address to pgt_info structure.
+ * @mmu_phys_hash: holds a mapping from physical address to pgt_info structure.
+ * @mmu_shadow_hash: holds a mapping from shadow address to pgt_info structure.
* @hpriv: pointer to the private (KMD) data of the process (fd).
* @hdev: pointer to the device structure.
* @refcount: reference counter for the context. Context is released only when
@@ -601,17 +625,19 @@ struct hl_va_range {
* DRAM mapping.
* @cs_lock: spinlock to protect cs_sequence.
* @dram_phys_mem: amount of used physical DRAM memory by this context.
- * @thread_restore_token: token to prevent multiple threads of the same context
- * from running the restore phase. Only one thread
- * should run it.
- * @thread_restore_wait_token: token to prevent the threads that didn't run
- * the restore phase from moving to their execution
- * phase before the restore phase has finished.
+ * @thread_ctx_switch_token: token to prevent multiple threads of the same
+ * context from running the context switch phase.
+ * Only a single thread should run it.
+ * @thread_ctx_switch_wait_token: token to prevent the threads that didn't run
+ * the context switch phase from moving to their
+ * execution phase before the context switch phase
+ * has finished.
* @asid: context's unique address space ID in the device's MMU.
*/
struct hl_ctx {
DECLARE_HASHTABLE(mem_hash, MEM_HASH_TABLE_BITS);
- DECLARE_HASHTABLE(mmu_hash, MMU_HASH_TABLE_BITS);
+ DECLARE_HASHTABLE(mmu_phys_hash, MMU_HASH_TABLE_BITS);
+ DECLARE_HASHTABLE(mmu_shadow_hash, MMU_HASH_TABLE_BITS);
struct hl_fpriv *hpriv;
struct hl_device *hdev;
struct kref refcount;
@@ -625,8 +651,8 @@ struct hl_ctx {
u64 *dram_default_hops;
spinlock_t cs_lock;
atomic64_t dram_phys_mem;
- atomic_t thread_restore_token;
- u32 thread_restore_wait_token;
+ atomic_t thread_ctx_switch_token;
+ u32 thread_ctx_switch_wait_token;
u32 asid;
};
@@ -753,8 +779,6 @@ struct hl_cs_job {
* @patched_cb_size: the size of the CB after parsing.
* @ext_queue: whether the job is for external queue or internal queue.
* @job_id: the id of the related job inside the related CS.
- * @use_virt_addr: whether to treat the addresses in the CB as virtual during
- * parsing.
*/
struct hl_cs_parser {
struct hl_cb *user_cb;
@@ -767,7 +791,6 @@ struct hl_cs_parser {
u32 patched_cb_size;
u8 ext_queue;
u8 job_id;
- u8 use_virt_addr;
};
@@ -850,6 +873,29 @@ struct hl_vm {
u8 init_done;
};
+
+/*
+ * DEBUG, PROFILING STRUCTURE
+ */
+
+/**
+ * struct hl_debug_params - Coresight debug parameters.
+ * @input: pointer to component specific input parameters.
+ * @output: pointer to component specific output parameters.
+ * @output_size: size of output buffer.
+ * @reg_idx: relevant register ID.
+ * @op: component operation to execute.
+ * @enable: true if to enable component debugging, false otherwise.
+ */
+struct hl_debug_params {
+ void *input;
+ void *output;
+ u32 output_size;
+ u32 reg_idx;
+ u32 op;
+ bool enable;
+};
+
/*
* FILE PRIVATE STRUCTURE
*/
@@ -973,13 +1019,10 @@ struct hl_dbg_device_entry {
u32 hl_rreg(struct hl_device *hdev, u32 reg);
void hl_wreg(struct hl_device *hdev, u32 reg, u32 val);
-#define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \
- readl_poll_timeout(hdev->rmmio + addr, val, cond, sleep_us, timeout_us)
-
-#define RREG32(reg) hl_rreg(hdev, (reg))
-#define WREG32(reg, v) hl_wreg(hdev, (reg), (v))
+#define RREG32(reg) hdev->asic_funcs->rreg(hdev, (reg))
+#define WREG32(reg, v) hdev->asic_funcs->wreg(hdev, (reg), (v))
#define DREG32(reg) pr_info("REGISTER: " #reg " : 0x%08X\n", \
- hl_rreg(hdev, (reg)))
+ hdev->asic_funcs->rreg(hdev, (reg)))
#define WREG32_P(reg, val, mask) \
do { \
@@ -997,6 +1040,36 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val);
WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | \
(val) << REG_FIELD_SHIFT(reg, field))
+#define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \
+({ \
+ ktime_t __timeout; \
+ /* timeout should be longer when working with simulator */ \
+ if (hdev->pdev) \
+ __timeout = ktime_add_us(ktime_get(), timeout_us); \
+ else \
+ __timeout = ktime_add_us(ktime_get(), (timeout_us * 10)); \
+ might_sleep_if(sleep_us); \
+ for (;;) { \
+ (val) = RREG32(addr); \
+ if (cond) \
+ break; \
+ if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
+ (val) = RREG32(addr); \
+ break; \
+ } \
+ if (sleep_us) \
+ usleep_range((sleep_us >> 2) + 1, sleep_us); \
+ } \
+ (cond) ? 0 : -ETIMEDOUT; \
+})
+
+
+#define HL_ENG_BUSY(buf, size, fmt, ...) ({ \
+ if (buf) \
+ snprintf(buf, size, fmt, ##__VA_ARGS__); \
+ false; \
+ })
+
struct hwmon_chip_info;
/**
@@ -1047,7 +1120,8 @@ struct hl_device_reset_work {
* @asic_specific: ASIC specific information to use only from ASIC files.
* @mmu_pgt_pool: pool of available MMU hops.
* @vm: virtual memory manager for MMU.
- * @mmu_cache_lock: protects MMU cache invalidation as it can serve one context
+ * @mmu_cache_lock: protects MMU cache invalidation as it can serve one context.
+ * @mmu_shadow_hop0: shadow mapping of the MMU hop 0 zone.
* @hwmon_dev: H/W monitor device.
* @pm_mng_profile: current power management profile.
* @hl_chip_info: ASIC's sensors information.
@@ -1082,6 +1156,7 @@ struct hl_device_reset_work {
* @init_done: is the initialization of the device done.
* @mmu_enable: is MMU enabled.
* @device_cpu_disabled: is the device CPU disabled (due to timeouts)
+ * @dma_mask: the dma mask that was set for this device
*/
struct hl_device {
struct pci_dev *pdev;
@@ -1117,6 +1192,7 @@ struct hl_device {
struct gen_pool *mmu_pgt_pool;
struct hl_vm vm;
struct mutex mmu_cache_lock;
+ void *mmu_shadow_hop0;
struct device *hwmon_dev;
enum hl_pm_mng_profile pm_mng_profile;
struct hwmon_chip_info *hl_chip_info;
@@ -1151,6 +1227,7 @@ struct hl_device {
u8 dram_default_page_mapping;
u8 init_done;
u8 device_cpu_disabled;
+ u8 dma_mask;
/* Parameters for bring-up */
u8 mmu_enable;
@@ -1245,6 +1322,7 @@ static inline bool hl_mem_area_crosses_range(u64 address, u32 size,
int hl_device_open(struct inode *inode, struct file *filp);
bool hl_device_disabled_or_in_reset(struct hl_device *hdev);
+enum hl_device_status hl_device_status(struct hl_device *hdev);
int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
enum hl_asic_type asic_type, int minor);
void destroy_hdev(struct hl_device *hdev);
@@ -1351,6 +1429,32 @@ int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size);
void hl_mmu_swap_out(struct hl_ctx *ctx);
void hl_mmu_swap_in(struct hl_ctx *ctx);
+int hl_fw_push_fw_to_device(struct hl_device *hdev, const char *fw_name,
+ void __iomem *dst);
+int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode);
+int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
+ u16 len, u32 timeout, long *result);
+int hl_fw_test_cpu_queue(struct hl_device *hdev);
+void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
+ dma_addr_t *dma_handle);
+void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
+ void *vaddr);
+int hl_fw_send_heartbeat(struct hl_device *hdev);
+int hl_fw_armcp_info_get(struct hl_device *hdev);
+int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size);
+
+int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
+ bool is_wc[3]);
+int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data);
+int hl_pci_set_dram_bar_base(struct hl_device *hdev, u8 inbound_region, u8 bar,
+ u64 addr);
+int hl_pci_init_iatu(struct hl_device *hdev, u64 sram_base_address,
+ u64 dram_base_address, u64 host_phys_base_address,
+ u64 host_phys_size);
+int hl_pci_init(struct hl_device *hdev, u8 dma_mask);
+void hl_pci_fini(struct hl_device *hdev);
+int hl_pci_set_dma_mask(struct hl_device *hdev, u8 dma_mask);
+
long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr);
void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq);
long hl_get_temperature(struct hl_device *hdev, int sensor_index, u32 attr);
diff --git a/drivers/misc/habanalabs/habanalabs_drv.c b/drivers/misc/habanalabs/habanalabs_drv.c
index 748601463f11..5f4d155be767 100644
--- a/drivers/misc/habanalabs/habanalabs_drv.c
+++ b/drivers/misc/habanalabs/habanalabs_drv.c
@@ -6,6 +6,8 @@
*
*/
+#define pr_fmt(fmt) "habanalabs: " fmt
+
#include "habanalabs.h"
#include <linux/pci.h>
@@ -218,7 +220,7 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
hdev->disabled = true;
hdev->pdev = pdev; /* can be NULL in case of simulator device */
- if (asic_type == ASIC_AUTO_DETECT) {
+ if (pdev) {
hdev->asic_type = get_asic_type(pdev->device);
if (hdev->asic_type == ASIC_INVALID) {
dev_err(&pdev->dev, "Unsupported ASIC\n");
@@ -229,6 +231,9 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
hdev->asic_type = asic_type;
}
+ /* Set default DMA mask to 32 bits */
+ hdev->dma_mask = 32;
+
mutex_lock(&hl_devs_idr_lock);
if (minor == -1) {
@@ -334,7 +339,7 @@ static int hl_pci_probe(struct pci_dev *pdev,
" device found [%04x:%04x] (rev %x)\n",
(int)pdev->vendor, (int)pdev->device, (int)pdev->revision);
- rc = create_hdev(&hdev, pdev, ASIC_AUTO_DETECT, -1);
+ rc = create_hdev(&hdev, pdev, ASIC_INVALID, -1);
if (rc)
return rc;
diff --git a/drivers/misc/habanalabs/habanalabs_ioctl.c b/drivers/misc/habanalabs/habanalabs_ioctl.c
index 2c2739a3c5ec..eeefb22023e9 100644
--- a/drivers/misc/habanalabs/habanalabs_ioctl.c
+++ b/drivers/misc/habanalabs/habanalabs_ioctl.c
@@ -12,6 +12,32 @@
#include <linux/uaccess.h>
#include <linux/slab.h>
+static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = {
+ [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr),
+ [HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf),
+ [HL_DEBUG_OP_STM] = sizeof(struct hl_debug_params_stm),
+ [HL_DEBUG_OP_FUNNEL] = 0,
+ [HL_DEBUG_OP_BMON] = sizeof(struct hl_debug_params_bmon),
+ [HL_DEBUG_OP_SPMU] = sizeof(struct hl_debug_params_spmu),
+ [HL_DEBUG_OP_TIMESTAMP] = 0
+
+};
+
+static int device_status_info(struct hl_device *hdev, struct hl_info_args *args)
+{
+ struct hl_info_device_status dev_stat = {0};
+ u32 size = args->return_size;
+ void __user *out = (void __user *) (uintptr_t) args->return_pointer;
+
+ if ((!size) || (!out))
+ return -EINVAL;
+
+ dev_stat.status = hl_device_status(hdev);
+
+ return copy_to_user(out, &dev_stat,
+ min((size_t)size, sizeof(dev_stat))) ? -EFAULT : 0;
+}
+
static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
{
struct hl_info_hw_ip_info hw_ip = {0};
@@ -93,21 +119,91 @@ static int hw_idle(struct hl_device *hdev, struct hl_info_args *args)
if ((!max_size) || (!out))
return -EINVAL;
- hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev);
+ hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev, NULL, 0);
return copy_to_user(out, &hw_idle,
min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0;
}
+static int debug_coresight(struct hl_device *hdev, struct hl_debug_args *args)
+{
+ struct hl_debug_params *params;
+ void *input = NULL, *output = NULL;
+ int rc;
+
+ params = kzalloc(sizeof(*params), GFP_KERNEL);
+ if (!params)
+ return -ENOMEM;
+
+ params->reg_idx = args->reg_idx;
+ params->enable = args->enable;
+ params->op = args->op;
+
+ if (args->input_ptr && args->input_size) {
+ input = memdup_user((const void __user *) args->input_ptr,
+ args->input_size);
+ if (IS_ERR(input)) {
+ rc = PTR_ERR(input);
+ input = NULL;
+ dev_err(hdev->dev,
+ "error %d when copying input debug data\n", rc);
+ goto out;
+ }
+
+ params->input = input;
+ }
+
+ if (args->output_ptr && args->output_size) {
+ output = kzalloc(args->output_size, GFP_KERNEL);
+ if (!output) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ params->output = output;
+ params->output_size = args->output_size;
+ }
+
+ rc = hdev->asic_funcs->debug_coresight(hdev, params);
+ if (rc) {
+ dev_err(hdev->dev,
+ "debug coresight operation failed %d\n", rc);
+ goto out;
+ }
+
+ if (output) {
+ if (copy_to_user((void __user *) (uintptr_t) args->output_ptr,
+ output,
+ args->output_size)) {
+ dev_err(hdev->dev,
+ "copy to user failed in debug ioctl\n");
+ rc = -EFAULT;
+ goto out;
+ }
+ }
+
+out:
+ kfree(params);
+ kfree(output);
+ kfree(input);
+
+ return rc;
+}
+
static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
{
struct hl_info_args *args = data;
struct hl_device *hdev = hpriv->hdev;
int rc;
+ /* We want to return device status even if it disabled or in reset */
+ if (args->op == HL_INFO_DEVICE_STATUS)
+ return device_status_info(hdev, args);
+
if (hl_device_disabled_or_in_reset(hdev)) {
- dev_err(hdev->dev,
- "Device is disabled or in reset. Can't execute INFO IOCTL\n");
+ dev_warn_ratelimited(hdev->dev,
+ "Device is %s. Can't execute INFO IOCTL\n",
+ atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
return -EBUSY;
}
@@ -137,6 +233,40 @@ static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
return rc;
}
+static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
+{
+ struct hl_debug_args *args = data;
+ struct hl_device *hdev = hpriv->hdev;
+ int rc = 0;
+
+ if (hl_device_disabled_or_in_reset(hdev)) {
+ dev_warn_ratelimited(hdev->dev,
+ "Device is %s. Can't execute DEBUG IOCTL\n",
+ atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
+ return -EBUSY;
+ }
+
+ switch (args->op) {
+ case HL_DEBUG_OP_ETR:
+ case HL_DEBUG_OP_ETF:
+ case HL_DEBUG_OP_STM:
+ case HL_DEBUG_OP_FUNNEL:
+ case HL_DEBUG_OP_BMON:
+ case HL_DEBUG_OP_SPMU:
+ case HL_DEBUG_OP_TIMESTAMP:
+ args->input_size =
+ min(args->input_size, hl_debug_struct_size[args->op]);
+ rc = debug_coresight(hdev, args);
+ break;
+ default:
+ dev_err(hdev->dev, "Invalid request %d\n", args->op);
+ rc = -ENOTTY;
+ break;
+ }
+
+ return rc;
+}
+
#define HL_IOCTL_DEF(ioctl, _func) \
[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func}
@@ -145,7 +275,8 @@ static const struct hl_ioctl_desc hl_ioctls[] = {
HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_cs_wait_ioctl),
- HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl)
+ HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
+ HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
};
#define HL_CORE_IOCTL_COUNT ARRAY_SIZE(hl_ioctls)
diff --git a/drivers/misc/habanalabs/hw_queue.c b/drivers/misc/habanalabs/hw_queue.c
index ef3bb6951360..2894d8975933 100644
--- a/drivers/misc/habanalabs/hw_queue.c
+++ b/drivers/misc/habanalabs/hw_queue.c
@@ -82,7 +82,7 @@ static void ext_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,
bd += hl_pi_2_offset(q->pi);
bd->ctl = __cpu_to_le32(ctl);
bd->len = __cpu_to_le32(len);
- bd->ptr = __cpu_to_le64(ptr + hdev->asic_prop.host_phys_base_address);
+ bd->ptr = __cpu_to_le64(ptr);
q->pi = hl_queue_inc_ptr(q->pi);
hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
@@ -263,9 +263,7 @@ static void ext_hw_queue_schedule_job(struct hl_cs_job *job)
* checked in hl_queue_sanity_checks
*/
cq = &hdev->completion_queue[q->hw_queue_id];
- cq_addr = cq->bus_address +
- hdev->asic_prop.host_phys_base_address;
- cq_addr += cq->pi * sizeof(struct hl_cq_entry);
+ cq_addr = cq->bus_address + cq->pi * sizeof(struct hl_cq_entry);
hdev->asic_funcs->add_end_of_cb_packets(cb->kernel_address, len,
cq_addr,
@@ -415,14 +413,20 @@ void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)
}
static int ext_and_cpu_hw_queue_init(struct hl_device *hdev,
- struct hl_hw_queue *q)
+ struct hl_hw_queue *q, bool is_cpu_queue)
{
void *p;
int rc;
- p = hdev->asic_funcs->dma_alloc_coherent(hdev,
- HL_QUEUE_SIZE_IN_BYTES,
- &q->bus_address, GFP_KERNEL | __GFP_ZERO);
+ if (is_cpu_queue)
+ p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
+ HL_QUEUE_SIZE_IN_BYTES,
+ &q->bus_address);
+ else
+ p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
+ HL_QUEUE_SIZE_IN_BYTES,
+ &q->bus_address,
+ GFP_KERNEL | __GFP_ZERO);
if (!p)
return -ENOMEM;
@@ -446,8 +450,15 @@ static int ext_and_cpu_hw_queue_init(struct hl_device *hdev,
return 0;
free_queue:
- hdev->asic_funcs->dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES,
- (void *) (uintptr_t) q->kernel_address, q->bus_address);
+ if (is_cpu_queue)
+ hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
+ HL_QUEUE_SIZE_IN_BYTES,
+ (void *) (uintptr_t) q->kernel_address);
+ else
+ hdev->asic_funcs->asic_dma_free_coherent(hdev,
+ HL_QUEUE_SIZE_IN_BYTES,
+ (void *) (uintptr_t) q->kernel_address,
+ q->bus_address);
return rc;
}
@@ -474,12 +485,12 @@ static int int_hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
static int cpu_hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
{
- return ext_and_cpu_hw_queue_init(hdev, q);
+ return ext_and_cpu_hw_queue_init(hdev, q, true);
}
static int ext_hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
{
- return ext_and_cpu_hw_queue_init(hdev, q);
+ return ext_and_cpu_hw_queue_init(hdev, q, false);
}
/*
@@ -569,8 +580,15 @@ static void hw_queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)
kfree(q->shadow_queue);
- hdev->asic_funcs->dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES,
- (void *) (uintptr_t) q->kernel_address, q->bus_address);
+ if (q->queue_type == QUEUE_TYPE_CPU)
+ hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
+ HL_QUEUE_SIZE_IN_BYTES,
+ (void *) (uintptr_t) q->kernel_address);
+ else
+ hdev->asic_funcs->asic_dma_free_coherent(hdev,
+ HL_QUEUE_SIZE_IN_BYTES,
+ (void *) (uintptr_t) q->kernel_address,
+ q->bus_address);
}
int hl_hw_queues_create(struct hl_device *hdev)
diff --git a/drivers/misc/habanalabs/include/armcp_if.h b/drivers/misc/habanalabs/include/armcp_if.h
index 9dddb917e72c..1f1e35e86d84 100644
--- a/drivers/misc/habanalabs/include/armcp_if.h
+++ b/drivers/misc/habanalabs/include/armcp_if.h
@@ -32,8 +32,6 @@ struct hl_eq_entry {
#define EQ_CTL_EVENT_TYPE_SHIFT 16
#define EQ_CTL_EVENT_TYPE_MASK 0x03FF0000
-#define EVENT_QUEUE_MSIX_IDX 5
-
enum pq_init_status {
PQ_INIT_STATUS_NA = 0,
PQ_INIT_STATUS_READY_FOR_CP,
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/cpu_ca53_cfg_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/cpu_ca53_cfg_masks.h
index 2cf5c46b6e8e..4e0dbbbbde20 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/cpu_ca53_cfg_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/cpu_ca53_cfg_masks.h
@@ -188,4 +188,3 @@
#define CPU_CA53_CFG_ARM_PMU_EVENT_MASK 0x3FFFFFFF
#endif /* ASIC_REG_CPU_CA53_CFG_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/cpu_ca53_cfg_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/cpu_ca53_cfg_regs.h
index 840ccffa1081..f3faf1aad91a 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/cpu_ca53_cfg_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/cpu_ca53_cfg_regs.h
@@ -58,4 +58,3 @@
#define mmCPU_CA53_CFG_ARM_PMU_1 0x441214
#endif /* ASIC_REG_CPU_CA53_CFG_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/cpu_if_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/cpu_if_regs.h
index f23cb3e41c30..cf657918962a 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/cpu_if_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/cpu_if_regs.h
@@ -46,4 +46,3 @@
#define mmCPU_IF_AXI_SPLIT_INTR 0x442130
#endif /* ASIC_REG_CPU_IF_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/cpu_pll_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/cpu_pll_regs.h
index 8fc97f838ada..8c8f9726d4b9 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/cpu_pll_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/cpu_pll_regs.h
@@ -102,4 +102,3 @@
#define mmCPU_PLL_FREQ_CALC_EN 0x4A2440
#endif /* ASIC_REG_CPU_PLL_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_0_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_0_regs.h
index 61c8cd9ce58b..0b246fe6ad04 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_0_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_0_regs.h
@@ -206,4 +206,3 @@
#define mmDMA_CH_0_MEM_INIT_BUSY 0x4011FC
#endif /* ASIC_REG_DMA_CH_0_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_1_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_1_regs.h
index 92960ef5e308..5449031722f2 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_1_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_1_regs.h
@@ -206,4 +206,3 @@
#define mmDMA_CH_1_MEM_INIT_BUSY 0x4091FC
#endif /* ASIC_REG_DMA_CH_1_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_2_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_2_regs.h
index 4e37871a51bb..a4768521d18a 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_2_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_2_regs.h
@@ -206,4 +206,3 @@
#define mmDMA_CH_2_MEM_INIT_BUSY 0x4111FC
#endif /* ASIC_REG_DMA_CH_2_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_3_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_3_regs.h
index a2d6aeb32a18..619d01897ff8 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_3_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_3_regs.h
@@ -206,4 +206,3 @@
#define mmDMA_CH_3_MEM_INIT_BUSY 0x4191FC
#endif /* ASIC_REG_DMA_CH_3_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_4_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_4_regs.h
index 400d6fd3acf5..038617e163f1 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_4_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_ch_4_regs.h
@@ -206,4 +206,3 @@
#define mmDMA_CH_4_MEM_INIT_BUSY 0x4211FC
#endif /* ASIC_REG_DMA_CH_4_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_macro_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_macro_masks.h
index 8d965443c51e..f43b564af1be 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_macro_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_macro_masks.h
@@ -102,4 +102,3 @@
#define DMA_MACRO_RAZWI_HBW_RD_ID_R_MASK 0x1FFFFFFF
#endif /* ASIC_REG_DMA_MACRO_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_macro_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_macro_regs.h
index 8bfcb001189d..c3bfc1b8e3fd 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_macro_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_macro_regs.h
@@ -178,4 +178,3 @@
#define mmDMA_MACRO_RAZWI_HBW_RD_ID 0x4B0158
#endif /* ASIC_REG_DMA_MACRO_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_nrtr_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_nrtr_masks.h
index 9f33f351a3c1..bc977488c072 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_nrtr_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_nrtr_masks.h
@@ -206,4 +206,3 @@
#define DMA_NRTR_NON_LIN_SCRAMB_EN_MASK 0x1
#endif /* ASIC_REG_DMA_NRTR_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_nrtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_nrtr_regs.h
index d8293745a02b..c4abc7ff1fc6 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_nrtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_nrtr_regs.h
@@ -224,4 +224,3 @@
#define mmDMA_NRTR_NON_LIN_SCRAMB 0x1C0604
#endif /* ASIC_REG_DMA_NRTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_0_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_0_masks.h
index 10619dbb9b17..b17f72c31ab6 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_0_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_0_masks.h
@@ -462,4 +462,3 @@
#define DMA_QM_0_CQ_BUF_RDATA_VAL_MASK 0xFFFFFFFF
#endif /* ASIC_REG_DMA_QM_0_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_0_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_0_regs.h
index c693bc5dcb22..bf360b301154 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_0_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_0_regs.h
@@ -176,4 +176,3 @@
#define mmDMA_QM_0_CQ_BUF_RDATA 0x40030C
#endif /* ASIC_REG_DMA_QM_0_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_1_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_1_regs.h
index da928390f89c..51d432d05ac4 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_1_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_1_regs.h
@@ -176,4 +176,3 @@
#define mmDMA_QM_1_CQ_BUF_RDATA 0x40830C
#endif /* ASIC_REG_DMA_QM_1_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_2_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_2_regs.h
index b4f06e9b71d6..18fc0c2b6cc2 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_2_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_2_regs.h
@@ -176,4 +176,3 @@
#define mmDMA_QM_2_CQ_BUF_RDATA 0x41030C
#endif /* ASIC_REG_DMA_QM_2_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_3_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_3_regs.h
index 53e3cd78a06b..6cf7204bf5cc 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_3_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_3_regs.h
@@ -176,4 +176,3 @@
#define mmDMA_QM_3_CQ_BUF_RDATA 0x41830C
#endif /* ASIC_REG_DMA_QM_3_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_4_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_4_regs.h
index e0eb5f260201..36fef2682875 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_4_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/dma_qm_4_regs.h
@@ -176,4 +176,3 @@
#define mmDMA_QM_4_CQ_BUF_RDATA 0x42030C
#endif /* ASIC_REG_DMA_QM_4_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h
index a161ecfe74de..8618891d5afa 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h
@@ -189,18 +189,6 @@
1 << CPU_CA53_CFG_ARM_RST_CONTROL_NL2RESET_SHIFT |\
1 << CPU_CA53_CFG_ARM_RST_CONTROL_NMBISTRESET_SHIFT)
-/* PCI CONFIGURATION SPACE */
-#define mmPCI_CONFIG_ELBI_ADDR 0xFF0
-#define mmPCI_CONFIG_ELBI_DATA 0xFF4
-#define mmPCI_CONFIG_ELBI_CTRL 0xFF8
-#define PCI_CONFIG_ELBI_CTRL_WRITE (1 << 31)
-
-#define mmPCI_CONFIG_ELBI_STS 0xFFC
-#define PCI_CONFIG_ELBI_STS_ERR (1 << 30)
-#define PCI_CONFIG_ELBI_STS_DONE (1 << 31)
-#define PCI_CONFIG_ELBI_STS_MASK (PCI_CONFIG_ELBI_STS_ERR | \
- PCI_CONFIG_ELBI_STS_DONE)
-
#define GOYA_IRQ_HBW_ID_MASK 0x1FFF
#define GOYA_IRQ_HBW_ID_SHIFT 0
#define GOYA_IRQ_HBW_INTERNAL_ID_MASK 0xE000
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h
index 6cb0b6e54d41..506e71e201e1 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h
@@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0
*
- * Copyright 2016-2018 HabanaLabs, Ltd.
+ * Copyright 2016-2019 HabanaLabs, Ltd.
* All Rights Reserved.
*
*/
@@ -12,6 +12,7 @@
#include "stlb_regs.h"
#include "mmu_regs.h"
#include "pcie_aux_regs.h"
+#include "pcie_wrap_regs.h"
#include "psoc_global_conf_regs.h"
#include "psoc_spi_regs.h"
#include "psoc_mme_pll_regs.h"
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/ic_pll_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/ic_pll_regs.h
index 0a743817aad7..4ae7fed8b18c 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/ic_pll_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/ic_pll_regs.h
@@ -102,4 +102,3 @@
#define mmIC_PLL_FREQ_CALC_EN 0x4A3440
#endif /* ASIC_REG_IC_PLL_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mc_pll_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mc_pll_regs.h
index 4408188aa067..6d35d852798b 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mc_pll_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mc_pll_regs.h
@@ -102,4 +102,3 @@
#define mmMC_PLL_FREQ_CALC_EN 0x4A1440
#endif /* ASIC_REG_MC_PLL_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme1_rtr_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme1_rtr_masks.h
index 687bca5c5fe3..6c23f8b96e7e 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme1_rtr_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme1_rtr_masks.h
@@ -650,4 +650,3 @@
#define MME1_RTR_NON_LIN_SCRAMB_EN_MASK 0x1
#endif /* ASIC_REG_MME1_RTR_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme1_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme1_rtr_regs.h
index c248339a1cbe..122e9d529939 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme1_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme1_rtr_regs.h
@@ -328,4 +328,3 @@
#define mmMME1_RTR_NON_LIN_SCRAMB 0x40604
#endif /* ASIC_REG_MME1_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme2_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme2_rtr_regs.h
index 7a2b777bdc4f..00ce2252bbfb 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme2_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme2_rtr_regs.h
@@ -328,4 +328,3 @@
#define mmMME2_RTR_NON_LIN_SCRAMB 0x80604
#endif /* ASIC_REG_MME2_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme3_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme3_rtr_regs.h
index b78f8bc387fc..8e3eb7fd2070 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme3_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme3_rtr_regs.h
@@ -328,4 +328,3 @@
#define mmMME3_RTR_NON_LIN_SCRAMB 0xC0604
#endif /* ASIC_REG_MME3_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme4_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme4_rtr_regs.h
index d9a4a02cefa3..79b67bbc8567 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme4_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme4_rtr_regs.h
@@ -328,4 +328,3 @@
#define mmMME4_RTR_NON_LIN_SCRAMB 0x100604
#endif /* ASIC_REG_MME4_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme5_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme5_rtr_regs.h
index 205adc988407..0ac3c37ce47f 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme5_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme5_rtr_regs.h
@@ -328,4 +328,3 @@
#define mmMME5_RTR_NON_LIN_SCRAMB 0x140604
#endif /* ASIC_REG_MME5_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme6_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme6_rtr_regs.h
index fcec68388278..50c49cce72a6 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme6_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme6_rtr_regs.h
@@ -328,4 +328,3 @@
#define mmMME6_RTR_NON_LIN_SCRAMB 0x180604
#endif /* ASIC_REG_MME6_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme_cmdq_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme_cmdq_masks.h
index a0d4382fbbd0..fe7d95bdcef9 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme_cmdq_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme_cmdq_masks.h
@@ -370,4 +370,3 @@
#define MME_CMDQ_CQ_BUF_RDATA_VAL_MASK 0xFFFFFFFF
#endif /* ASIC_REG_MME_CMDQ_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme_cmdq_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme_cmdq_regs.h
index 5c2f6b870a58..5f8b85d2b4b1 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme_cmdq_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme_cmdq_regs.h
@@ -136,4 +136,3 @@
#define mmMME_CMDQ_CQ_BUF_RDATA 0xD930C
#endif /* ASIC_REG_MME_CMDQ_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme_masks.h
index c7b1b0bb3384..1882c413cbe0 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme_masks.h
@@ -1534,4 +1534,3 @@
#define MME_SHADOW_3_E_BUBBLES_PER_SPLIT_ID_MASK 0xFF000000
#endif /* ASIC_REG_MME_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme_qm_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme_qm_masks.h
index d4bfa58dce19..e464e381555c 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme_qm_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme_qm_masks.h
@@ -462,4 +462,3 @@
#define MME_QM_CQ_BUF_RDATA_VAL_MASK 0xFFFFFFFF
#endif /* ASIC_REG_MME_QM_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme_qm_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme_qm_regs.h
index b5b1c776f6c3..538708beffc9 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme_qm_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme_qm_regs.h
@@ -176,4 +176,3 @@
#define mmMME_QM_CQ_BUF_RDATA 0xD830C
#endif /* ASIC_REG_MME_QM_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mme_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mme_regs.h
index 9436b1e2705a..0396cbfd5c89 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mme_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mme_regs.h
@@ -1150,4 +1150,3 @@
#define mmMME_SHADOW_3_E_BUBBLES_PER_SPLIT 0xD0BAC
#endif /* ASIC_REG_MME_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mmu_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/mmu_masks.h
index 3a78078d3c4c..c3e69062b135 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mmu_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mmu_masks.h
@@ -140,4 +140,3 @@
#define MMU_ACCESS_ERROR_CAPTURE_VA_VA_31_0_MASK 0xFFFFFFFF
#endif /* ASIC_REG_MMU_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/mmu_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/mmu_regs.h
index bec6c014135c..7ec81f12031e 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/mmu_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/mmu_regs.h
@@ -50,4 +50,3 @@
#define mmMMU_ACCESS_ERROR_CAPTURE_VA 0x480040
#endif /* ASIC_REG_MMU_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/pci_nrtr_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/pci_nrtr_masks.h
index 209e41402a11..ceb59f2e28b3 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/pci_nrtr_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/pci_nrtr_masks.h
@@ -206,4 +206,3 @@
#define PCI_NRTR_NON_LIN_SCRAMB_EN_MASK 0x1
#endif /* ASIC_REG_PCI_NRTR_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/pci_nrtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/pci_nrtr_regs.h
index 447e5d4e7dc8..dd067f301ac2 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/pci_nrtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/pci_nrtr_regs.h
@@ -224,4 +224,3 @@
#define mmPCI_NRTR_NON_LIN_SCRAMB 0x604
#endif /* ASIC_REG_PCI_NRTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/pcie_aux_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/pcie_aux_regs.h
index daaf5d9079dc..35b1d8ac6f63 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/pcie_aux_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/pcie_aux_regs.h
@@ -240,4 +240,3 @@
#define mmPCIE_AUX_PERST 0xC079B8
#endif /* ASIC_REG_PCIE_AUX_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/pcie_wrap_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/pcie_wrap_regs.h
new file mode 100644
index 000000000000..d1e55aace4a0
--- /dev/null
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/pcie_wrap_regs.h
@@ -0,0 +1,306 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ ** DO NOT EDIT BELOW **
+ ************************************/
+
+#ifndef ASIC_REG_PCIE_WRAP_REGS_H_
+#define ASIC_REG_PCIE_WRAP_REGS_H_
+
+/*
+ *****************************************
+ * PCIE_WRAP (Prototype: PCIE_WRAP)
+ *****************************************
+ */
+
+#define mmPCIE_WRAP_PHY_RST_N 0xC01300
+
+#define mmPCIE_WRAP_OUTSTAND_TRANS 0xC01400
+
+#define mmPCIE_WRAP_MASK_REQ 0xC01404
+
+#define mmPCIE_WRAP_IND_AWADDR_L 0xC01500
+
+#define mmPCIE_WRAP_IND_AWADDR_H 0xC01504
+
+#define mmPCIE_WRAP_IND_AWLEN 0xC01508
+
+#define mmPCIE_WRAP_IND_AWSIZE 0xC0150C
+
+#define mmPCIE_WRAP_IND_AWBURST 0xC01510
+
+#define mmPCIE_WRAP_IND_AWLOCK 0xC01514
+
+#define mmPCIE_WRAP_IND_AWCACHE 0xC01518
+
+#define mmPCIE_WRAP_IND_AWPROT 0xC0151C
+
+#define mmPCIE_WRAP_IND_AWVALID 0xC01520
+
+#define mmPCIE_WRAP_IND_WDATA_0 0xC01524
+
+#define mmPCIE_WRAP_IND_WDATA_1 0xC01528
+
+#define mmPCIE_WRAP_IND_WDATA_2 0xC0152C
+
+#define mmPCIE_WRAP_IND_WDATA_3 0xC01530
+
+#define mmPCIE_WRAP_IND_WSTRB 0xC01544
+
+#define mmPCIE_WRAP_IND_WLAST 0xC01548
+
+#define mmPCIE_WRAP_IND_WVALID 0xC0154C
+
+#define mmPCIE_WRAP_IND_BRESP 0xC01550
+
+#define mmPCIE_WRAP_IND_BVALID 0xC01554
+
+#define mmPCIE_WRAP_IND_ARADDR_0 0xC01558
+
+#define mmPCIE_WRAP_IND_ARADDR_1 0xC0155C
+
+#define mmPCIE_WRAP_IND_ARLEN 0xC01560
+
+#define mmPCIE_WRAP_IND_ARSIZE 0xC01564
+
+#define mmPCIE_WRAP_IND_ARBURST 0xC01568
+
+#define mmPCIE_WRAP_IND_ARLOCK 0xC0156C
+
+#define mmPCIE_WRAP_IND_ARCACHE 0xC01570
+
+#define mmPCIE_WRAP_IND_ARPROT 0xC01574
+
+#define mmPCIE_WRAP_IND_ARVALID 0xC01578
+
+#define mmPCIE_WRAP_IND_RDATA_0 0xC0157C
+
+#define mmPCIE_WRAP_IND_RDATA_1 0xC01580
+
+#define mmPCIE_WRAP_IND_RDATA_2 0xC01584
+
+#define mmPCIE_WRAP_IND_RDATA_3 0xC01588
+
+#define mmPCIE_WRAP_IND_RLAST 0xC0159C
+
+#define mmPCIE_WRAP_IND_RRESP 0xC015A0
+
+#define mmPCIE_WRAP_IND_RVALID 0xC015A4
+
+#define mmPCIE_WRAP_IND_AWMISC_INFO 0xC015A8
+
+#define mmPCIE_WRAP_IND_AWMISC_INFO_HDR_34DW_0 0xC015AC
+
+#define mmPCIE_WRAP_IND_AWMISC_INFO_HDR_34DW_1 0xC015B0
+
+#define mmPCIE_WRAP_IND_AWMISC_INFO_P_TAG 0xC015B4
+
+#define mmPCIE_WRAP_IND_AWMISC_INFO_ATU_BYPAS 0xC015B8
+
+#define mmPCIE_WRAP_IND_AWMISC_INFO_FUNC_NUM 0xC015BC
+
+#define mmPCIE_WRAP_IND_AWMISC_INFO_VFUNC_ACT 0xC015C0
+
+#define mmPCIE_WRAP_IND_AWMISC_INFO_VFUNC_NUM 0xC015C4
+
+#define mmPCIE_WRAP_IND_AWMISC_INFO_TLPPRFX 0xC015C8
+
+#define mmPCIE_WRAP_IND_ARMISC_INFO 0xC015CC
+
+#define mmPCIE_WRAP_IND_ARMISC_INFO_TLPPRFX 0xC015D0
+
+#define mmPCIE_WRAP_IND_ARMISC_INFO_ATU_BYP 0xC015D4
+
+#define mmPCIE_WRAP_IND_ARMISC_INFO_FUNC_NUM 0xC015D8
+
+#define mmPCIE_WRAP_IND_ARMISC_INFO_VFUNC_ACT 0xC015DC
+
+#define mmPCIE_WRAP_IND_ARMISC_INFO_VFUNC_NUM 0xC015E0
+
+#define mmPCIE_WRAP_SLV_AWMISC_INFO 0xC01800
+
+#define mmPCIE_WRAP_SLV_AWMISC_INFO_HDR_34DW_0 0xC01804
+
+#define mmPCIE_WRAP_SLV_AWMISC_INFO_HDR_34DW_1 0xC01808
+
+#define mmPCIE_WRAP_SLV_AWMISC_INFO_P_TAG 0xC0180C
+
+#define mmPCIE_WRAP_SLV_AWMISC_INFO_ATU_BYPAS 0xC01810
+
+#define mmPCIE_WRAP_SLV_AWMISC_INFO_FUNC_NUM 0xC01814
+
+#define mmPCIE_WRAP_SLV_AWMISC_INFO_VFUNC_ACT 0xC01818
+
+#define mmPCIE_WRAP_SLV_AWMISC_INFO_VFUNC_NUM 0xC0181C
+
+#define mmPCIE_WRAP_SLV_AWMISC_INFO_TLPPRFX 0xC01820
+
+#define mmPCIE_WRAP_SLV_ARMISC_INFO 0xC01824
+
+#define mmPCIE_WRAP_SLV_ARMISC_INFO_TLPPRFX 0xC01828
+
+#define mmPCIE_WRAP_SLV_ARMISC_INFO_ATU_BYP 0xC0182C
+
+#define mmPCIE_WRAP_SLV_ARMISC_INFO_FUNC_NUM 0xC01830
+
+#define mmPCIE_WRAP_SLV_ARMISC_INFO_VFUNC_ACT 0xC01834
+
+#define mmPCIE_WRAP_SLV_ARMISC_INFO_VFUNC_NUM 0xC01838
+
+#define mmPCIE_WRAP_MAX_QID 0xC01900
+
+#define mmPCIE_WRAP_DB_BASE_ADDR_L_0 0xC01910
+
+#define mmPCIE_WRAP_DB_BASE_ADDR_L_1 0xC01914
+
+#define mmPCIE_WRAP_DB_BASE_ADDR_L_2 0xC01918
+
+#define mmPCIE_WRAP_DB_BASE_ADDR_L_3 0xC0191C
+
+#define mmPCIE_WRAP_DB_BASE_ADDR_H_0 0xC01920
+
+#define mmPCIE_WRAP_DB_BASE_ADDR_H_1 0xC01924
+
+#define mmPCIE_WRAP_DB_BASE_ADDR_H_2 0xC01928
+
+#define mmPCIE_WRAP_DB_BASE_ADDR_H_3 0xC0192C
+
+#define mmPCIE_WRAP_DB_MASK 0xC01940
+
+#define mmPCIE_WRAP_SQ_BASE_ADDR_H 0xC01A00
+
+#define mmPCIE_WRAP_SQ_BASE_ADDR_L 0xC01A04
+
+#define mmPCIE_WRAP_SQ_STRIDE_ACCRESS 0xC01A08
+
+#define mmPCIE_WRAP_SQ_POP_CMD 0xC01A10
+
+#define mmPCIE_WRAP_SQ_POP_DATA 0xC01A14
+
+#define mmPCIE_WRAP_DB_INTR_0 0xC01A20
+
+#define mmPCIE_WRAP_DB_INTR_1 0xC01A24
+
+#define mmPCIE_WRAP_DB_INTR_2 0xC01A28
+
+#define mmPCIE_WRAP_DB_INTR_3 0xC01A2C
+
+#define mmPCIE_WRAP_DB_INTR_4 0xC01A30
+
+#define mmPCIE_WRAP_DB_INTR_5 0xC01A34
+
+#define mmPCIE_WRAP_DB_INTR_6 0xC01A38
+
+#define mmPCIE_WRAP_DB_INTR_7 0xC01A3C
+
+#define mmPCIE_WRAP_MMU_BYPASS_DMA 0xC01A80
+
+#define mmPCIE_WRAP_MMU_BYPASS_NON_DMA 0xC01A84
+
+#define mmPCIE_WRAP_ASID_NON_DMA 0xC01A90
+
+#define mmPCIE_WRAP_ASID_DMA_0 0xC01AA0
+
+#define mmPCIE_WRAP_ASID_DMA_1 0xC01AA4
+
+#define mmPCIE_WRAP_ASID_DMA_2 0xC01AA8
+
+#define mmPCIE_WRAP_ASID_DMA_3 0xC01AAC
+
+#define mmPCIE_WRAP_ASID_DMA_4 0xC01AB0
+
+#define mmPCIE_WRAP_ASID_DMA_5 0xC01AB4
+
+#define mmPCIE_WRAP_ASID_DMA_6 0xC01AB8
+
+#define mmPCIE_WRAP_ASID_DMA_7 0xC01ABC
+
+#define mmPCIE_WRAP_CPU_HOT_RST 0xC01AE0
+
+#define mmPCIE_WRAP_AXI_PROT_OVR 0xC01AE4
+
+#define mmPCIE_WRAP_CACHE_OVR 0xC01B00
+
+#define mmPCIE_WRAP_LOCK_OVR 0xC01B04
+
+#define mmPCIE_WRAP_PROT_OVR 0xC01B08
+
+#define mmPCIE_WRAP_ARUSER_OVR 0xC01B0C
+
+#define mmPCIE_WRAP_AWUSER_OVR 0xC01B10
+
+#define mmPCIE_WRAP_ARUSER_OVR_EN 0xC01B14
+
+#define mmPCIE_WRAP_AWUSER_OVR_EN 0xC01B18
+
+#define mmPCIE_WRAP_MAX_OUTSTAND 0xC01B20
+
+#define mmPCIE_WRAP_MST_IN 0xC01B24
+
+#define mmPCIE_WRAP_RSP_OK 0xC01B28
+
+#define mmPCIE_WRAP_LBW_CACHE_OVR 0xC01B40
+
+#define mmPCIE_WRAP_LBW_LOCK_OVR 0xC01B44
+
+#define mmPCIE_WRAP_LBW_PROT_OVR 0xC01B48
+
+#define mmPCIE_WRAP_LBW_ARUSER_OVR 0xC01B4C
+
+#define mmPCIE_WRAP_LBW_AWUSER_OVR 0xC01B50
+
+#define mmPCIE_WRAP_LBW_ARUSER_OVR_EN 0xC01B58
+
+#define mmPCIE_WRAP_LBW_AWUSER_OVR_EN 0xC01B5C
+
+#define mmPCIE_WRAP_LBW_MAX_OUTSTAND 0xC01B60
+
+#define mmPCIE_WRAP_LBW_MST_IN 0xC01B64
+
+#define mmPCIE_WRAP_LBW_RSP_OK 0xC01B68
+
+#define mmPCIE_WRAP_QUEUE_INIT 0xC01C00
+
+#define mmPCIE_WRAP_AXI_SPLIT_INTR_0 0xC01C10
+
+#define mmPCIE_WRAP_AXI_SPLIT_INTR_1 0xC01C14
+
+#define mmPCIE_WRAP_DB_AWUSER 0xC01D00
+
+#define mmPCIE_WRAP_DB_ARUSER 0xC01D04
+
+#define mmPCIE_WRAP_PCIE_AWUSER 0xC01D08
+
+#define mmPCIE_WRAP_PCIE_ARUSER 0xC01D0C
+
+#define mmPCIE_WRAP_PSOC_AWUSER 0xC01D10
+
+#define mmPCIE_WRAP_PSOC_ARUSER 0xC01D14
+
+#define mmPCIE_WRAP_SCH_Q_AWUSER 0xC01D18
+
+#define mmPCIE_WRAP_SCH_Q_ARUSER 0xC01D1C
+
+#define mmPCIE_WRAP_PSOC2PCI_AWUSER 0xC01D40
+
+#define mmPCIE_WRAP_PSOC2PCI_ARUSER 0xC01D44
+
+#define mmPCIE_WRAP_DRAIN_TIMEOUT 0xC01D50
+
+#define mmPCIE_WRAP_DRAIN_CFG 0xC01D54
+
+#define mmPCIE_WRAP_DB_AXI_ERR 0xC01DE0
+
+#define mmPCIE_WRAP_SPMU_INTR 0xC01DE4
+
+#define mmPCIE_WRAP_AXI_INTR 0xC01DE8
+
+#define mmPCIE_WRAP_E2E_CTRL 0xC01DF0
+
+#endif /* ASIC_REG_PCIE_WRAP_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_emmc_pll_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_emmc_pll_regs.h
index 8eda4de58788..9271ea95ebe9 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_emmc_pll_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_emmc_pll_regs.h
@@ -102,4 +102,3 @@
#define mmPSOC_EMMC_PLL_FREQ_CALC_EN 0xC70440
#endif /* ASIC_REG_PSOC_EMMC_PLL_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_global_conf_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_global_conf_masks.h
index d4bf0e1db4df..324266653c9a 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_global_conf_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_global_conf_masks.h
@@ -444,4 +444,3 @@
#define PSOC_GLOBAL_CONF_PAD_SEL_VAL_MASK 0x3
#endif /* ASIC_REG_PSOC_GLOBAL_CONF_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_global_conf_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_global_conf_regs.h
index cfbdd2c9c5c7..8141f422e712 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_global_conf_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_global_conf_regs.h
@@ -742,4 +742,3 @@
#define mmPSOC_GLOBAL_CONF_PAD_SEL_81 0xC4BA44
#endif /* ASIC_REG_PSOC_GLOBAL_CONF_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_mme_pll_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_mme_pll_regs.h
index 6723d8f76f30..4789ebb9c337 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_mme_pll_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_mme_pll_regs.h
@@ -102,4 +102,3 @@
#define mmPSOC_MME_PLL_FREQ_CALC_EN 0xC71440
#endif /* ASIC_REG_PSOC_MME_PLL_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_pci_pll_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_pci_pll_regs.h
index abcded0531c9..27a296ea6c3d 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_pci_pll_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_pci_pll_regs.h
@@ -102,4 +102,3 @@
#define mmPSOC_PCI_PLL_FREQ_CALC_EN 0xC72440
#endif /* ASIC_REG_PSOC_PCI_PLL_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_spi_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_spi_regs.h
index 5925c7477c25..66aee7fa6b1e 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_spi_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_spi_regs.h
@@ -140,4 +140,3 @@
#define mmPSOC_SPI_RSVD_2 0xC430FC
#endif /* ASIC_REG_PSOC_SPI_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x0_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x0_rtr_regs.h
index d56c9fa0e7ba..2ea1770b078f 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x0_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x0_rtr_regs.h
@@ -80,4 +80,3 @@
#define mmSRAM_Y0_X0_RTR_DBG_L_ARB_MAX 0x201330
#endif /* ASIC_REG_SRAM_Y0_X0_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x1_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x1_rtr_regs.h
index 5624544303ca..37e0713efa73 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x1_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x1_rtr_regs.h
@@ -80,4 +80,3 @@
#define mmSRAM_Y0_X1_RTR_DBG_L_ARB_MAX 0x205330
#endif /* ASIC_REG_SRAM_Y0_X1_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x2_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x2_rtr_regs.h
index 3322bc0bd1df..d2572279a2b9 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x2_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x2_rtr_regs.h
@@ -80,4 +80,3 @@
#define mmSRAM_Y0_X2_RTR_DBG_L_ARB_MAX 0x209330
#endif /* ASIC_REG_SRAM_Y0_X2_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x3_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x3_rtr_regs.h
index 81e393db2027..68c5b402c506 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x3_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x3_rtr_regs.h
@@ -80,4 +80,3 @@
#define mmSRAM_Y0_X3_RTR_DBG_L_ARB_MAX 0x20D330
#endif /* ASIC_REG_SRAM_Y0_X3_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x4_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x4_rtr_regs.h
index b2e11b1de385..a42f1ba06d28 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x4_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/sram_y0_x4_rtr_regs.h
@@ -80,4 +80,3 @@
#define mmSRAM_Y0_X4_RTR_DBG_L_ARB_MAX 0x211330
#endif /* ASIC_REG_SRAM_Y0_X4_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/stlb_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/stlb_masks.h
index b4ea8cae2757..94f2ed4a36bd 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/stlb_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/stlb_masks.h
@@ -114,4 +114,3 @@
#define STLB_SRAM_INIT_BUSY_DATA_MASK 0x10
#endif /* ASIC_REG_STLB_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/stlb_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/stlb_regs.h
index 0f5281d3e65b..35013f65acd2 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/stlb_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/stlb_regs.h
@@ -52,4 +52,3 @@
#define mmSTLB_SRAM_INIT 0x49004C
#endif /* ASIC_REG_STLB_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cfg_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cfg_masks.h
index e5587b49eecd..89c9507a512f 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cfg_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cfg_masks.h
@@ -1604,4 +1604,3 @@
#define TPC0_CFG_FUNC_MBIST_MEM_LAST_FAILED_PATTERN_MASK 0x70000000
#endif /* ASIC_REG_TPC0_CFG_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cfg_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cfg_regs.h
index 2be28a63c50a..7d71c4b73a5e 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cfg_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cfg_regs.h
@@ -884,4 +884,3 @@
#define mmTPC0_CFG_FUNC_MBIST_MEM_9 0xE06E2C
#endif /* ASIC_REG_TPC0_CFG_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cmdq_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cmdq_masks.h
index 9aa2d8b53207..9395f2458771 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cmdq_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cmdq_masks.h
@@ -370,4 +370,3 @@
#define TPC0_CMDQ_CQ_BUF_RDATA_VAL_MASK 0xFFFFFFFF
#endif /* ASIC_REG_TPC0_CMDQ_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cmdq_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cmdq_regs.h
index 3572752ba66e..bc51df573bf0 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cmdq_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_cmdq_regs.h
@@ -136,4 +136,3 @@
#define mmTPC0_CMDQ_CQ_BUF_RDATA 0xE0930C
#endif /* ASIC_REG_TPC0_CMDQ_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_eml_cfg_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_eml_cfg_masks.h
index ed866d93c440..553c6b6bd5ec 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_eml_cfg_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_eml_cfg_masks.h
@@ -344,4 +344,3 @@
#define TPC0_EML_CFG_DBG_INST_INSERT_CTL_INSERT_MASK 0x1
#endif /* ASIC_REG_TPC0_EML_CFG_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_eml_cfg_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_eml_cfg_regs.h
index f1a1b4fa4841..8495479c3659 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_eml_cfg_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_eml_cfg_regs.h
@@ -310,4 +310,3 @@
#define mmTPC0_EML_CFG_DBG_INST_INSERT_CTL 0x3040334
#endif /* ASIC_REG_TPC0_EML_CFG_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_nrtr_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_nrtr_masks.h
index 7f86621179a5..43fafcf01041 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_nrtr_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_nrtr_masks.h
@@ -206,4 +206,3 @@
#define TPC0_NRTR_NON_LIN_SCRAMB_EN_MASK 0x1
#endif /* ASIC_REG_TPC0_NRTR_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_nrtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_nrtr_regs.h
index dc280f4e6608..ce3346dd2042 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_nrtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_nrtr_regs.h
@@ -224,4 +224,3 @@
#define mmTPC0_NRTR_NON_LIN_SCRAMB 0xE00604
#endif /* ASIC_REG_TPC0_NRTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_qm_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_qm_masks.h
index 80d97ee3d8d6..2e4b45947944 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_qm_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_qm_masks.h
@@ -462,4 +462,3 @@
#define TPC0_QM_CQ_BUF_RDATA_VAL_MASK 0xFFFFFFFF
#endif /* ASIC_REG_TPC0_QM_MASKS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_qm_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_qm_regs.h
index 7552d4ba61fe..4fa09eb88878 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_qm_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc0_qm_regs.h
@@ -176,4 +176,3 @@
#define mmTPC0_QM_CQ_BUF_RDATA 0xE0830C
#endif /* ASIC_REG_TPC0_QM_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_cfg_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_cfg_regs.h
index 19894413474a..928eef1808ae 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_cfg_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_cfg_regs.h
@@ -884,4 +884,3 @@
#define mmTPC1_CFG_FUNC_MBIST_MEM_9 0xE46E2C
#endif /* ASIC_REG_TPC1_CFG_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_cmdq_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_cmdq_regs.h
index 9099ebd7ab23..30ae0f307328 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_cmdq_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_cmdq_regs.h
@@ -136,4 +136,3 @@
#define mmTPC1_CMDQ_CQ_BUF_RDATA 0xE4930C
#endif /* ASIC_REG_TPC1_CMDQ_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_qm_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_qm_regs.h
index bc8b9a10391f..b95de4f95ba9 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_qm_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_qm_regs.h
@@ -176,4 +176,3 @@
#define mmTPC1_QM_CQ_BUF_RDATA 0xE4830C
#endif /* ASIC_REG_TPC1_QM_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_rtr_regs.h
index ae267f8f457e..0f91e307879e 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc1_rtr_regs.h
@@ -320,4 +320,3 @@
#define mmTPC1_RTR_NON_LIN_SCRAMB 0xE40604
#endif /* ASIC_REG_TPC1_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_cfg_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_cfg_regs.h
index 9c33fc039036..73421227f35b 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_cfg_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_cfg_regs.h
@@ -884,4 +884,3 @@
#define mmTPC2_CFG_FUNC_MBIST_MEM_9 0xE86E2C
#endif /* ASIC_REG_TPC2_CFG_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_cmdq_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_cmdq_regs.h
index 7a643887d6e1..27b66bf2da9f 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_cmdq_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_cmdq_regs.h
@@ -136,4 +136,3 @@
#define mmTPC2_CMDQ_CQ_BUF_RDATA 0xE8930C
#endif /* ASIC_REG_TPC2_CMDQ_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_qm_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_qm_regs.h
index f3e32c018064..31e5b2f53905 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_qm_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_qm_regs.h
@@ -176,4 +176,3 @@
#define mmTPC2_QM_CQ_BUF_RDATA 0xE8830C
#endif /* ASIC_REG_TPC2_QM_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_rtr_regs.h
index 0eb0cd1fbd19..4eddeaa15d94 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc2_rtr_regs.h
@@ -320,4 +320,3 @@
#define mmTPC2_RTR_NON_LIN_SCRAMB 0xE80604
#endif /* ASIC_REG_TPC2_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_cfg_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_cfg_regs.h
index 0baf63c69b25..ce573a1a8361 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_cfg_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_cfg_regs.h
@@ -884,4 +884,3 @@
#define mmTPC3_CFG_FUNC_MBIST_MEM_9 0xEC6E2C
#endif /* ASIC_REG_TPC3_CFG_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_cmdq_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_cmdq_regs.h
index 82a5261e852f..11d81fca0a0f 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_cmdq_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_cmdq_regs.h
@@ -136,4 +136,3 @@
#define mmTPC3_CMDQ_CQ_BUF_RDATA 0xEC930C
#endif /* ASIC_REG_TPC3_CMDQ_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_qm_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_qm_regs.h
index b05b1e18e664..e41595a19e69 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_qm_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_qm_regs.h
@@ -176,4 +176,3 @@
#define mmTPC3_QM_CQ_BUF_RDATA 0xEC830C
#endif /* ASIC_REG_TPC3_QM_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_rtr_regs.h
index 5a2fd7652650..34a438b1efe5 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc3_rtr_regs.h
@@ -320,4 +320,3 @@
#define mmTPC3_RTR_NON_LIN_SCRAMB 0xEC0604
#endif /* ASIC_REG_TPC3_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_cfg_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_cfg_regs.h
index d64a100075f2..d44caf0fc1bb 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_cfg_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_cfg_regs.h
@@ -884,4 +884,3 @@
#define mmTPC4_CFG_FUNC_MBIST_MEM_9 0xF06E2C
#endif /* ASIC_REG_TPC4_CFG_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_cmdq_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_cmdq_regs.h
index 565b42885b0d..f13a6532961f 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_cmdq_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_cmdq_regs.h
@@ -136,4 +136,3 @@
#define mmTPC4_CMDQ_CQ_BUF_RDATA 0xF0930C
#endif /* ASIC_REG_TPC4_CMDQ_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_qm_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_qm_regs.h
index 196da3f12710..db081fc17cfc 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_qm_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_qm_regs.h
@@ -176,4 +176,3 @@
#define mmTPC4_QM_CQ_BUF_RDATA 0xF0830C
#endif /* ASIC_REG_TPC4_QM_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_rtr_regs.h
index 8b54041d144a..8c5372303b28 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc4_rtr_regs.h
@@ -320,4 +320,3 @@
#define mmTPC4_RTR_NON_LIN_SCRAMB 0xF00604
#endif /* ASIC_REG_TPC4_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_cfg_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_cfg_regs.h
index 3f00954fcdba..5139fde71011 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_cfg_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_cfg_regs.h
@@ -884,4 +884,3 @@
#define mmTPC5_CFG_FUNC_MBIST_MEM_9 0xF46E2C
#endif /* ASIC_REG_TPC5_CFG_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_cmdq_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_cmdq_regs.h
index d8e72a8e18d7..1e7cd6e1e888 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_cmdq_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_cmdq_regs.h
@@ -136,4 +136,3 @@
#define mmTPC5_CMDQ_CQ_BUF_RDATA 0xF4930C
#endif /* ASIC_REG_TPC5_CMDQ_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_qm_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_qm_regs.h
index be2e68624709..ac0d3820cd6b 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_qm_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_qm_regs.h
@@ -176,4 +176,3 @@
#define mmTPC5_QM_CQ_BUF_RDATA 0xF4830C
#endif /* ASIC_REG_TPC5_QM_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_rtr_regs.h
index 6f301c7bbc2f..57f83bc3b17d 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc5_rtr_regs.h
@@ -320,4 +320,3 @@
#define mmTPC5_RTR_NON_LIN_SCRAMB 0xF40604
#endif /* ASIC_REG_TPC5_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_cfg_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_cfg_regs.h
index 1e1168601c41..94e0191c06c1 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_cfg_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_cfg_regs.h
@@ -884,4 +884,3 @@
#define mmTPC6_CFG_FUNC_MBIST_MEM_9 0xF86E2C
#endif /* ASIC_REG_TPC6_CFG_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_cmdq_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_cmdq_regs.h
index fbca6b47284e..7a1a0e87b225 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_cmdq_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_cmdq_regs.h
@@ -136,4 +136,3 @@
#define mmTPC6_CMDQ_CQ_BUF_RDATA 0xF8930C
#endif /* ASIC_REG_TPC6_CMDQ_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_qm_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_qm_regs.h
index bf32465dabcb..80fa0fe0f60f 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_qm_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_qm_regs.h
@@ -176,4 +176,3 @@
#define mmTPC6_QM_CQ_BUF_RDATA 0xF8830C
#endif /* ASIC_REG_TPC6_QM_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_rtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_rtr_regs.h
index 609bb90e1046..d6cae8b8af66 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_rtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc6_rtr_regs.h
@@ -320,4 +320,3 @@
#define mmTPC6_RTR_NON_LIN_SCRAMB 0xF80604
#endif /* ASIC_REG_TPC6_RTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_cfg_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_cfg_regs.h
index bf2fd0f73906..234147adb779 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_cfg_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_cfg_regs.h
@@ -884,4 +884,3 @@
#define mmTPC7_CFG_FUNC_MBIST_MEM_9 0xFC6E2C
#endif /* ASIC_REG_TPC7_CFG_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_cmdq_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_cmdq_regs.h
index 65d83043bf63..4c160632fe7d 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_cmdq_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_cmdq_regs.h
@@ -136,4 +136,3 @@
#define mmTPC7_CMDQ_CQ_BUF_RDATA 0xFC930C
#endif /* ASIC_REG_TPC7_CMDQ_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_nrtr_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_nrtr_regs.h
index 3d5848d87304..0c13d4d167aa 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_nrtr_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_nrtr_regs.h
@@ -224,4 +224,3 @@
#define mmTPC7_NRTR_NON_LIN_SCRAMB 0xFC0604
#endif /* ASIC_REG_TPC7_NRTR_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_qm_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_qm_regs.h
index 25f5095f68fb..cbe11425bfb0 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_qm_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc7_qm_regs.h
@@ -176,4 +176,3 @@
#define mmTPC7_QM_CQ_BUF_RDATA 0xFC830C
#endif /* ASIC_REG_TPC7_QM_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/tpc_pll_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/tpc_pll_regs.h
index 920231d0afa5..e25e19660a9d 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/tpc_pll_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/tpc_pll_regs.h
@@ -102,4 +102,3 @@
#define mmTPC_PLL_FREQ_CALC_EN 0xE01440
#endif /* ASIC_REG_TPC_PLL_REGS_H_ */
-
diff --git a/drivers/misc/habanalabs/include/goya/goya.h b/drivers/misc/habanalabs/include/goya/goya.h
index 614149efa412..3f02a52ba4ce 100644
--- a/drivers/misc/habanalabs/include/goya/goya.h
+++ b/drivers/misc/habanalabs/include/goya/goya.h
@@ -8,10 +8,6 @@
#ifndef GOYA_H
#define GOYA_H
-#include "asic_reg/goya_regs.h"
-
-#include <linux/types.h>
-
#define SRAM_CFG_BAR_ID 0
#define MSIX_BAR_ID 2
#define DDR_BAR_ID 4
diff --git a/drivers/misc/habanalabs/include/goya/goya_async_events.h b/drivers/misc/habanalabs/include/goya/goya_async_events.h
index 497937a17ee9..bb7a1aa3279e 100644
--- a/drivers/misc/habanalabs/include/goya/goya_async_events.h
+++ b/drivers/misc/habanalabs/include/goya/goya_async_events.h
@@ -9,7 +9,9 @@
#define __GOYA_ASYNC_EVENTS_H_
enum goya_async_event_id {
+ GOYA_ASYNC_EVENT_ID_PCIE_CORE = 32,
GOYA_ASYNC_EVENT_ID_PCIE_IF = 33,
+ GOYA_ASYNC_EVENT_ID_PCIE_PHY = 34,
GOYA_ASYNC_EVENT_ID_TPC0_ECC = 36,
GOYA_ASYNC_EVENT_ID_TPC1_ECC = 39,
GOYA_ASYNC_EVENT_ID_TPC2_ECC = 42,
@@ -23,6 +25,8 @@ enum goya_async_event_id {
GOYA_ASYNC_EVENT_ID_MMU_ECC = 63,
GOYA_ASYNC_EVENT_ID_DMA_MACRO = 64,
GOYA_ASYNC_EVENT_ID_DMA_ECC = 66,
+ GOYA_ASYNC_EVENT_ID_DDR0_PARITY = 69,
+ GOYA_ASYNC_EVENT_ID_DDR1_PARITY = 72,
GOYA_ASYNC_EVENT_ID_CPU_IF_ECC = 75,
GOYA_ASYNC_EVENT_ID_PSOC_MEM = 78,
GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT = 79,
@@ -72,6 +76,7 @@ enum goya_async_event_id {
GOYA_ASYNC_EVENT_ID_MME_WACSD = 142,
GOYA_ASYNC_EVENT_ID_PLL0 = 143,
GOYA_ASYNC_EVENT_ID_PLL1 = 144,
+ GOYA_ASYNC_EVENT_ID_PLL2 = 145,
GOYA_ASYNC_EVENT_ID_PLL3 = 146,
GOYA_ASYNC_EVENT_ID_PLL4 = 147,
GOYA_ASYNC_EVENT_ID_PLL5 = 148,
@@ -81,6 +86,7 @@ enum goya_async_event_id {
GOYA_ASYNC_EVENT_ID_PSOC = 160,
GOYA_ASYNC_EVENT_ID_PCIE_FLR = 171,
GOYA_ASYNC_EVENT_ID_PCIE_HOT_RESET = 172,
+ GOYA_ASYNC_EVENT_ID_PCIE_PERST = 173,
GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG0 = 174,
GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG1 = 175,
GOYA_ASYNC_EVENT_ID_PCIE_QID0_ENG2 = 176,
@@ -144,8 +150,11 @@ enum goya_async_event_id {
GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_0 = 330,
GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_1 = 331,
GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_2 = 332,
+ GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_3 = 333,
+ GOYA_ASYNC_EVENT_ID_PSOC_GPIO_U16_4 = 334,
GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET = 356,
GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT = 361,
+ GOYA_ASYNC_EVENT_ID_FAN = 425,
GOYA_ASYNC_EVENT_ID_TPC0_CMDQ = 430,
GOYA_ASYNC_EVENT_ID_TPC1_CMDQ = 431,
GOYA_ASYNC_EVENT_ID_TPC2_CMDQ = 432,
diff --git a/drivers/misc/habanalabs/include/goya/goya_coresight.h b/drivers/misc/habanalabs/include/goya/goya_coresight.h
new file mode 100644
index 000000000000..6e933c0ca5cd
--- /dev/null
+++ b/drivers/misc/habanalabs/include/goya/goya_coresight.h
@@ -0,0 +1,199 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef GOYA_CORESIGHT_H
+#define GOYA_CORESIGHT_H
+
+enum goya_debug_stm_regs_index {
+ GOYA_STM_FIRST = 0,
+ GOYA_STM_CPU = GOYA_STM_FIRST,
+ GOYA_STM_DMA_CH_0_CS,
+ GOYA_STM_DMA_CH_1_CS,
+ GOYA_STM_DMA_CH_2_CS,
+ GOYA_STM_DMA_CH_3_CS,
+ GOYA_STM_DMA_CH_4_CS,
+ GOYA_STM_DMA_MACRO_CS,
+ GOYA_STM_MME1_SBA,
+ GOYA_STM_MME3_SBB,
+ GOYA_STM_MME4_WACS2,
+ GOYA_STM_MME4_WACS,
+ GOYA_STM_MMU_CS,
+ GOYA_STM_PCIE,
+ GOYA_STM_PSOC,
+ GOYA_STM_TPC0_EML,
+ GOYA_STM_TPC1_EML,
+ GOYA_STM_TPC2_EML,
+ GOYA_STM_TPC3_EML,
+ GOYA_STM_TPC4_EML,
+ GOYA_STM_TPC5_EML,
+ GOYA_STM_TPC6_EML,
+ GOYA_STM_TPC7_EML,
+ GOYA_STM_LAST = GOYA_STM_TPC7_EML
+};
+
+enum goya_debug_etf_regs_index {
+ GOYA_ETF_FIRST = 0,
+ GOYA_ETF_CPU_0 = GOYA_ETF_FIRST,
+ GOYA_ETF_CPU_1,
+ GOYA_ETF_CPU_TRACE,
+ GOYA_ETF_DMA_CH_0_CS,
+ GOYA_ETF_DMA_CH_1_CS,
+ GOYA_ETF_DMA_CH_2_CS,
+ GOYA_ETF_DMA_CH_3_CS,
+ GOYA_ETF_DMA_CH_4_CS,
+ GOYA_ETF_DMA_MACRO_CS,
+ GOYA_ETF_MME1_SBA,
+ GOYA_ETF_MME3_SBB,
+ GOYA_ETF_MME4_WACS2,
+ GOYA_ETF_MME4_WACS,
+ GOYA_ETF_MMU_CS,
+ GOYA_ETF_PCIE,
+ GOYA_ETF_PSOC,
+ GOYA_ETF_TPC0_EML,
+ GOYA_ETF_TPC1_EML,
+ GOYA_ETF_TPC2_EML,
+ GOYA_ETF_TPC3_EML,
+ GOYA_ETF_TPC4_EML,
+ GOYA_ETF_TPC5_EML,
+ GOYA_ETF_TPC6_EML,
+ GOYA_ETF_TPC7_EML,
+ GOYA_ETF_LAST = GOYA_ETF_TPC7_EML
+};
+
+enum goya_debug_funnel_regs_index {
+ GOYA_FUNNEL_FIRST = 0,
+ GOYA_FUNNEL_CPU = GOYA_FUNNEL_FIRST,
+ GOYA_FUNNEL_DMA_CH_6_1,
+ GOYA_FUNNEL_DMA_MACRO_3_1,
+ GOYA_FUNNEL_MME0_RTR,
+ GOYA_FUNNEL_MME1_RTR,
+ GOYA_FUNNEL_MME2_RTR,
+ GOYA_FUNNEL_MME3_RTR,
+ GOYA_FUNNEL_MME4_RTR,
+ GOYA_FUNNEL_MME5_RTR,
+ GOYA_FUNNEL_PCIE,
+ GOYA_FUNNEL_PSOC,
+ GOYA_FUNNEL_TPC0_EML,
+ GOYA_FUNNEL_TPC1_EML,
+ GOYA_FUNNEL_TPC1_RTR,
+ GOYA_FUNNEL_TPC2_EML,
+ GOYA_FUNNEL_TPC2_RTR,
+ GOYA_FUNNEL_TPC3_EML,
+ GOYA_FUNNEL_TPC3_RTR,
+ GOYA_FUNNEL_TPC4_EML,
+ GOYA_FUNNEL_TPC4_RTR,
+ GOYA_FUNNEL_TPC5_EML,
+ GOYA_FUNNEL_TPC5_RTR,
+ GOYA_FUNNEL_TPC6_EML,
+ GOYA_FUNNEL_TPC6_RTR,
+ GOYA_FUNNEL_TPC7_EML,
+ GOYA_FUNNEL_LAST = GOYA_FUNNEL_TPC7_EML
+};
+
+enum goya_debug_bmon_regs_index {
+ GOYA_BMON_FIRST = 0,
+ GOYA_BMON_CPU_RD = GOYA_BMON_FIRST,
+ GOYA_BMON_CPU_WR,
+ GOYA_BMON_DMA_CH_0_0,
+ GOYA_BMON_DMA_CH_0_1,
+ GOYA_BMON_DMA_CH_1_0,
+ GOYA_BMON_DMA_CH_1_1,
+ GOYA_BMON_DMA_CH_2_0,
+ GOYA_BMON_DMA_CH_2_1,
+ GOYA_BMON_DMA_CH_3_0,
+ GOYA_BMON_DMA_CH_3_1,
+ GOYA_BMON_DMA_CH_4_0,
+ GOYA_BMON_DMA_CH_4_1,
+ GOYA_BMON_DMA_MACRO_0,
+ GOYA_BMON_DMA_MACRO_1,
+ GOYA_BMON_DMA_MACRO_2,
+ GOYA_BMON_DMA_MACRO_3,
+ GOYA_BMON_DMA_MACRO_4,
+ GOYA_BMON_DMA_MACRO_5,
+ GOYA_BMON_DMA_MACRO_6,
+ GOYA_BMON_DMA_MACRO_7,
+ GOYA_BMON_MME1_SBA_0,
+ GOYA_BMON_MME1_SBA_1,
+ GOYA_BMON_MME3_SBB_0,
+ GOYA_BMON_MME3_SBB_1,
+ GOYA_BMON_MME4_WACS2_0,
+ GOYA_BMON_MME4_WACS2_1,
+ GOYA_BMON_MME4_WACS2_2,
+ GOYA_BMON_MME4_WACS_0,
+ GOYA_BMON_MME4_WACS_1,
+ GOYA_BMON_MME4_WACS_2,
+ GOYA_BMON_MME4_WACS_3,
+ GOYA_BMON_MME4_WACS_4,
+ GOYA_BMON_MME4_WACS_5,
+ GOYA_BMON_MME4_WACS_6,
+ GOYA_BMON_MMU_0,
+ GOYA_BMON_MMU_1,
+ GOYA_BMON_PCIE_MSTR_RD,
+ GOYA_BMON_PCIE_MSTR_WR,
+ GOYA_BMON_PCIE_SLV_RD,
+ GOYA_BMON_PCIE_SLV_WR,
+ GOYA_BMON_TPC0_EML_0,
+ GOYA_BMON_TPC0_EML_1,
+ GOYA_BMON_TPC0_EML_2,
+ GOYA_BMON_TPC0_EML_3,
+ GOYA_BMON_TPC1_EML_0,
+ GOYA_BMON_TPC1_EML_1,
+ GOYA_BMON_TPC1_EML_2,
+ GOYA_BMON_TPC1_EML_3,
+ GOYA_BMON_TPC2_EML_0,
+ GOYA_BMON_TPC2_EML_1,
+ GOYA_BMON_TPC2_EML_2,
+ GOYA_BMON_TPC2_EML_3,
+ GOYA_BMON_TPC3_EML_0,
+ GOYA_BMON_TPC3_EML_1,
+ GOYA_BMON_TPC3_EML_2,
+ GOYA_BMON_TPC3_EML_3,
+ GOYA_BMON_TPC4_EML_0,
+ GOYA_BMON_TPC4_EML_1,
+ GOYA_BMON_TPC4_EML_2,
+ GOYA_BMON_TPC4_EML_3,
+ GOYA_BMON_TPC5_EML_0,
+ GOYA_BMON_TPC5_EML_1,
+ GOYA_BMON_TPC5_EML_2,
+ GOYA_BMON_TPC5_EML_3,
+ GOYA_BMON_TPC6_EML_0,
+ GOYA_BMON_TPC6_EML_1,
+ GOYA_BMON_TPC6_EML_2,
+ GOYA_BMON_TPC6_EML_3,
+ GOYA_BMON_TPC7_EML_0,
+ GOYA_BMON_TPC7_EML_1,
+ GOYA_BMON_TPC7_EML_2,
+ GOYA_BMON_TPC7_EML_3,
+ GOYA_BMON_LAST = GOYA_BMON_TPC7_EML_3
+};
+
+enum goya_debug_spmu_regs_index {
+ GOYA_SPMU_FIRST = 0,
+ GOYA_SPMU_DMA_CH_0_CS = GOYA_SPMU_FIRST,
+ GOYA_SPMU_DMA_CH_1_CS,
+ GOYA_SPMU_DMA_CH_2_CS,
+ GOYA_SPMU_DMA_CH_3_CS,
+ GOYA_SPMU_DMA_CH_4_CS,
+ GOYA_SPMU_DMA_MACRO_CS,
+ GOYA_SPMU_MME1_SBA,
+ GOYA_SPMU_MME3_SBB,
+ GOYA_SPMU_MME4_WACS2,
+ GOYA_SPMU_MME4_WACS,
+ GOYA_SPMU_MMU_CS,
+ GOYA_SPMU_PCIE,
+ GOYA_SPMU_TPC0_EML,
+ GOYA_SPMU_TPC1_EML,
+ GOYA_SPMU_TPC2_EML,
+ GOYA_SPMU_TPC3_EML,
+ GOYA_SPMU_TPC4_EML,
+ GOYA_SPMU_TPC5_EML,
+ GOYA_SPMU_TPC6_EML,
+ GOYA_SPMU_TPC7_EML,
+ GOYA_SPMU_LAST = GOYA_SPMU_TPC7_EML
+};
+
+#endif /* GOYA_CORESIGHT_H */
diff --git a/drivers/misc/habanalabs/include/goya/goya_fw_if.h b/drivers/misc/habanalabs/include/goya/goya_fw_if.h
index a9920cb4a07b..0fa80fe9f6cc 100644
--- a/drivers/misc/habanalabs/include/goya/goya_fw_if.h
+++ b/drivers/misc/habanalabs/include/goya/goya_fw_if.h
@@ -8,6 +8,8 @@
#ifndef GOYA_FW_IF_H
#define GOYA_FW_IF_H
+#define GOYA_EVENT_QUEUE_MSIX_IDX 5
+
#define CPU_BOOT_ADDR 0x7FF8040000ull
#define UBOOT_FW_OFFSET 0x100000 /* 1MB in SRAM */
diff --git a/drivers/misc/habanalabs/include/hl_boot_if.h b/drivers/misc/habanalabs/include/hl_boot_if.h
index 7475732b9996..4cd04c090285 100644
--- a/drivers/misc/habanalabs/include/hl_boot_if.h
+++ b/drivers/misc/habanalabs/include/hl_boot_if.h
@@ -18,7 +18,8 @@ enum cpu_boot_status {
CPU_BOOT_STATUS_IN_SPL,
CPU_BOOT_STATUS_IN_UBOOT,
CPU_BOOT_STATUS_DRAM_INIT_FAIL,
- CPU_BOOT_STATUS_FIT_CORRUPTED
+ CPU_BOOT_STATUS_FIT_CORRUPTED,
+ CPU_BOOT_STATUS_UBOOT_NOT_READY,
};
enum kmd_msg {
diff --git a/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h b/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h
index b680052ee3f0..71ea3c3e8ba3 100644
--- a/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h
+++ b/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h
@@ -14,16 +14,16 @@
#define PAGE_SIZE_4KB (_AC(1, UL) << PAGE_SHIFT_4KB)
#define PAGE_MASK_2MB (~(PAGE_SIZE_2MB - 1))
-#define PAGE_PRESENT_MASK 0x0000000000001
-#define SWAP_OUT_MASK 0x0000000000004
-#define LAST_MASK 0x0000000000800
-#define PHYS_ADDR_MASK 0x3FFFFFFFFF000ull
+#define PAGE_PRESENT_MASK 0x0000000000001ull
+#define SWAP_OUT_MASK 0x0000000000004ull
+#define LAST_MASK 0x0000000000800ull
+#define PHYS_ADDR_MASK 0xFFFFFFFFFFFFF000ull
#define HOP0_MASK 0x3000000000000ull
#define HOP1_MASK 0x0FF8000000000ull
#define HOP2_MASK 0x0007FC0000000ull
-#define HOP3_MASK 0x000003FE00000
-#define HOP4_MASK 0x00000001FF000
-#define OFFSET_MASK 0x0000000000FFF
+#define HOP3_MASK 0x000003FE00000ull
+#define HOP4_MASK 0x00000001FF000ull
+#define OFFSET_MASK 0x0000000000FFFull
#define HOP0_SHIFT 48
#define HOP1_SHIFT 39
@@ -32,7 +32,7 @@
#define HOP4_SHIFT 12
#define PTE_PHYS_ADDR_SHIFT 12
-#define PTE_PHYS_ADDR_MASK ~0xFFF
+#define PTE_PHYS_ADDR_MASK ~OFFSET_MASK
#define HL_PTE_SIZE sizeof(u64)
#define HOP_TABLE_SIZE PAGE_SIZE_4KB
diff --git a/drivers/misc/habanalabs/include/hw_ip/pci/pci_general.h b/drivers/misc/habanalabs/include/hw_ip/pci/pci_general.h
new file mode 100644
index 000000000000..d232081d4e0f
--- /dev/null
+++ b/drivers/misc/habanalabs/include/hw_ip/pci/pci_general.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2019 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef INCLUDE_PCI_GENERAL_H_
+#define INCLUDE_PCI_GENERAL_H_
+
+/* PCI CONFIGURATION SPACE */
+#define mmPCI_CONFIG_ELBI_ADDR 0xFF0
+#define mmPCI_CONFIG_ELBI_DATA 0xFF4
+#define mmPCI_CONFIG_ELBI_CTRL 0xFF8
+#define PCI_CONFIG_ELBI_CTRL_WRITE (1 << 31)
+
+#define mmPCI_CONFIG_ELBI_STS 0xFFC
+#define PCI_CONFIG_ELBI_STS_ERR (1 << 30)
+#define PCI_CONFIG_ELBI_STS_DONE (1 << 31)
+#define PCI_CONFIG_ELBI_STS_MASK (PCI_CONFIG_ELBI_STS_ERR | \
+ PCI_CONFIG_ELBI_STS_DONE)
+
+#endif /* INCLUDE_PCI_GENERAL_H_ */
diff --git a/drivers/misc/habanalabs/irq.c b/drivers/misc/habanalabs/irq.c
index e69a09c10e3f..ea9f72ff456c 100644
--- a/drivers/misc/habanalabs/irq.c
+++ b/drivers/misc/habanalabs/irq.c
@@ -222,7 +222,7 @@ int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
BUILD_BUG_ON(HL_CQ_SIZE_IN_BYTES > HL_PAGE_SIZE);
- p = hdev->asic_funcs->dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
+ p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
&q->bus_address, GFP_KERNEL | __GFP_ZERO);
if (!p)
return -ENOMEM;
@@ -248,7 +248,7 @@ int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
*/
void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
{
- hdev->asic_funcs->dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
+ hdev->asic_funcs->asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
(void *) (uintptr_t) q->kernel_address, q->bus_address);
}
@@ -284,8 +284,9 @@ int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
BUILD_BUG_ON(HL_EQ_SIZE_IN_BYTES > HL_PAGE_SIZE);
- p = hdev->asic_funcs->dma_alloc_coherent(hdev, HL_EQ_SIZE_IN_BYTES,
- &q->bus_address, GFP_KERNEL | __GFP_ZERO);
+ p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
+ HL_EQ_SIZE_IN_BYTES,
+ &q->bus_address);
if (!p)
return -ENOMEM;
@@ -308,8 +309,9 @@ void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
{
flush_workqueue(hdev->eq_wq);
- hdev->asic_funcs->dma_free_coherent(hdev, HL_EQ_SIZE_IN_BYTES,
- (void *) (uintptr_t) q->kernel_address, q->bus_address);
+ hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
+ HL_EQ_SIZE_IN_BYTES,
+ (void *) (uintptr_t) q->kernel_address);
}
void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
diff --git a/drivers/misc/habanalabs/memory.c b/drivers/misc/habanalabs/memory.c
index ce1fda40a8b8..d67d24c13efd 100644
--- a/drivers/misc/habanalabs/memory.c
+++ b/drivers/misc/habanalabs/memory.c
@@ -109,7 +109,7 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args,
page_size);
if (!phys_pg_pack->pages[i]) {
dev_err(hdev->dev,
- "ioctl failed to allocate page\n");
+ "Failed to allocate device memory (out of memory)\n");
rc = -ENOMEM;
goto page_err;
}
@@ -759,10 +759,6 @@ static int map_phys_page_pack(struct hl_ctx *ctx, u64 vaddr,
for (i = 0 ; i < phys_pg_pack->npages ; i++) {
paddr = phys_pg_pack->pages[i];
- /* For accessing the host we need to turn on bit 39 */
- if (phys_pg_pack->created_from_userptr)
- paddr += hdev->asic_prop.host_phys_base_address;
-
rc = hl_mmu_map(ctx, next_vaddr, paddr, page_size);
if (rc) {
dev_err(hdev->dev,
@@ -1046,10 +1042,17 @@ static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr)
mutex_lock(&ctx->mmu_lock);
- for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size)
+ for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) {
if (hl_mmu_unmap(ctx, next_vaddr, page_size))
dev_warn_ratelimited(hdev->dev,
- "unmap failed for vaddr: 0x%llx\n", next_vaddr);
+ "unmap failed for vaddr: 0x%llx\n", next_vaddr);
+
+ /* unmapping on Palladium can be really long, so avoid a CPU
+ * soft lockup bug by sleeping a little between unmapping pages
+ */
+ if (hdev->pldm)
+ usleep_range(500, 1000);
+ }
hdev->asic_funcs->mmu_invalidate_cache(hdev, true);
@@ -1083,6 +1086,64 @@ vm_type_err:
return rc;
}
+static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args)
+{
+ struct hl_device *hdev = hpriv->hdev;
+ struct hl_ctx *ctx = hpriv->ctx;
+ u64 device_addr = 0;
+ u32 handle = 0;
+ int rc;
+
+ switch (args->in.op) {
+ case HL_MEM_OP_ALLOC:
+ if (args->in.alloc.mem_size == 0) {
+ dev_err(hdev->dev,
+ "alloc size must be larger than 0\n");
+ rc = -EINVAL;
+ goto out;
+ }
+
+ /* Force contiguous as there are no real MMU
+ * translations to overcome physical memory gaps
+ */
+ args->in.flags |= HL_MEM_CONTIGUOUS;
+ rc = alloc_device_memory(ctx, &args->in, &handle);
+
+ memset(args, 0, sizeof(*args));
+ args->out.handle = (__u64) handle;
+ break;
+
+ case HL_MEM_OP_FREE:
+ rc = free_device_memory(ctx, args->in.free.handle);
+ break;
+
+ case HL_MEM_OP_MAP:
+ if (args->in.flags & HL_MEM_USERPTR) {
+ device_addr = args->in.map_host.host_virt_addr;
+ rc = 0;
+ } else {
+ rc = get_paddr_from_handle(ctx, &args->in,
+ &device_addr);
+ }
+
+ memset(args, 0, sizeof(*args));
+ args->out.device_virt_addr = device_addr;
+ break;
+
+ case HL_MEM_OP_UNMAP:
+ rc = 0;
+ break;
+
+ default:
+ dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
+ rc = -ENOTTY;
+ break;
+ }
+
+out:
+ return rc;
+}
+
int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data)
{
union hl_mem_args *args = data;
@@ -1094,104 +1155,54 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data)
if (hl_device_disabled_or_in_reset(hdev)) {
dev_warn_ratelimited(hdev->dev,
- "Device is disabled or in reset. Can't execute memory IOCTL\n");
+ "Device is %s. Can't execute MEMORY IOCTL\n",
+ atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
return -EBUSY;
}
- if (hdev->mmu_enable) {
- switch (args->in.op) {
- case HL_MEM_OP_ALLOC:
- if (!hdev->dram_supports_virtual_memory) {
- dev_err(hdev->dev,
- "DRAM alloc is not supported\n");
- rc = -EINVAL;
- goto out;
- }
- if (args->in.alloc.mem_size == 0) {
- dev_err(hdev->dev,
- "alloc size must be larger than 0\n");
- rc = -EINVAL;
- goto out;
- }
- rc = alloc_device_memory(ctx, &args->in, &handle);
-
- memset(args, 0, sizeof(*args));
- args->out.handle = (__u64) handle;
- break;
-
- case HL_MEM_OP_FREE:
- if (!hdev->dram_supports_virtual_memory) {
- dev_err(hdev->dev,
- "DRAM free is not supported\n");
- rc = -EINVAL;
- goto out;
- }
- rc = free_device_memory(ctx, args->in.free.handle);
- break;
-
- case HL_MEM_OP_MAP:
- rc = map_device_va(ctx, &args->in, &device_addr);
-
- memset(args, 0, sizeof(*args));
- args->out.device_virt_addr = device_addr;
- break;
-
- case HL_MEM_OP_UNMAP:
- rc = unmap_device_va(ctx,
- args->in.unmap.device_virt_addr);
- break;
+ if (!hdev->mmu_enable)
+ return mem_ioctl_no_mmu(hpriv, args);
- default:
- dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
- rc = -ENOTTY;
- break;
+ switch (args->in.op) {
+ case HL_MEM_OP_ALLOC:
+ if (!hdev->dram_supports_virtual_memory) {
+ dev_err(hdev->dev, "DRAM alloc is not supported\n");
+ rc = -EINVAL;
+ goto out;
}
- } else {
- switch (args->in.op) {
- case HL_MEM_OP_ALLOC:
- if (args->in.alloc.mem_size == 0) {
- dev_err(hdev->dev,
- "alloc size must be larger than 0\n");
- rc = -EINVAL;
- goto out;
- }
- /* Force contiguous as there are no real MMU
- * translations to overcome physical memory gaps
- */
- args->in.flags |= HL_MEM_CONTIGUOUS;
- rc = alloc_device_memory(ctx, &args->in, &handle);
+ if (args->in.alloc.mem_size == 0) {
+ dev_err(hdev->dev,
+ "alloc size must be larger than 0\n");
+ rc = -EINVAL;
+ goto out;
+ }
+ rc = alloc_device_memory(ctx, &args->in, &handle);
- memset(args, 0, sizeof(*args));
- args->out.handle = (__u64) handle;
- break;
+ memset(args, 0, sizeof(*args));
+ args->out.handle = (__u64) handle;
+ break;
- case HL_MEM_OP_FREE:
- rc = free_device_memory(ctx, args->in.free.handle);
- break;
+ case HL_MEM_OP_FREE:
+ rc = free_device_memory(ctx, args->in.free.handle);
+ break;
- case HL_MEM_OP_MAP:
- if (args->in.flags & HL_MEM_USERPTR) {
- device_addr = args->in.map_host.host_virt_addr;
- rc = 0;
- } else {
- rc = get_paddr_from_handle(ctx, &args->in,
- &device_addr);
- }
+ case HL_MEM_OP_MAP:
+ rc = map_device_va(ctx, &args->in, &device_addr);
- memset(args, 0, sizeof(*args));
- args->out.device_virt_addr = device_addr;
- break;
+ memset(args, 0, sizeof(*args));
+ args->out.device_virt_addr = device_addr;
+ break;
- case HL_MEM_OP_UNMAP:
- rc = 0;
- break;
+ case HL_MEM_OP_UNMAP:
+ rc = unmap_device_va(ctx,
+ args->in.unmap.device_virt_addr);
+ break;
- default:
- dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
- rc = -ENOTTY;
- break;
- }
+ default:
+ dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n");
+ rc = -ENOTTY;
+ break;
}
out:
diff --git a/drivers/misc/habanalabs/mmu.c b/drivers/misc/habanalabs/mmu.c
index 3a5a2cec8305..533d9315b6fb 100644
--- a/drivers/misc/habanalabs/mmu.c
+++ b/drivers/misc/habanalabs/mmu.c
@@ -11,13 +11,15 @@
#include <linux/genalloc.h>
#include <linux/slab.h>
-static struct pgt_info *get_pgt_info(struct hl_ctx *ctx, u64 addr)
+static inline u64 get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr);
+
+static struct pgt_info *get_pgt_info(struct hl_ctx *ctx, u64 hop_addr)
{
struct pgt_info *pgt_info = NULL;
- hash_for_each_possible(ctx->mmu_hash, pgt_info, node,
- (unsigned long) addr)
- if (addr == pgt_info->addr)
+ hash_for_each_possible(ctx->mmu_shadow_hash, pgt_info, node,
+ (unsigned long) hop_addr)
+ if (hop_addr == pgt_info->shadow_addr)
break;
return pgt_info;
@@ -25,45 +27,109 @@ static struct pgt_info *get_pgt_info(struct hl_ctx *ctx, u64 addr)
static void free_hop(struct hl_ctx *ctx, u64 hop_addr)
{
+ struct hl_device *hdev = ctx->hdev;
struct pgt_info *pgt_info = get_pgt_info(ctx, hop_addr);
- gen_pool_free(pgt_info->ctx->hdev->mmu_pgt_pool, pgt_info->addr,
- ctx->hdev->asic_prop.mmu_hop_table_size);
+ gen_pool_free(hdev->mmu_pgt_pool, pgt_info->phys_addr,
+ hdev->asic_prop.mmu_hop_table_size);
hash_del(&pgt_info->node);
-
+ kfree((u64 *) (uintptr_t) pgt_info->shadow_addr);
kfree(pgt_info);
}
static u64 alloc_hop(struct hl_ctx *ctx)
{
struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
struct pgt_info *pgt_info;
- u64 addr;
+ u64 phys_addr, shadow_addr;
pgt_info = kmalloc(sizeof(*pgt_info), GFP_KERNEL);
if (!pgt_info)
return ULLONG_MAX;
- addr = (u64) gen_pool_alloc(hdev->mmu_pgt_pool,
- hdev->asic_prop.mmu_hop_table_size);
- if (!addr) {
+ phys_addr = (u64) gen_pool_alloc(hdev->mmu_pgt_pool,
+ prop->mmu_hop_table_size);
+ if (!phys_addr) {
dev_err(hdev->dev, "failed to allocate page\n");
- kfree(pgt_info);
- return ULLONG_MAX;
+ goto pool_add_err;
}
- pgt_info->addr = addr;
+ shadow_addr = (u64) (uintptr_t) kzalloc(prop->mmu_hop_table_size,
+ GFP_KERNEL);
+ if (!shadow_addr)
+ goto shadow_err;
+
+ pgt_info->phys_addr = phys_addr;
+ pgt_info->shadow_addr = shadow_addr;
pgt_info->ctx = ctx;
pgt_info->num_of_ptes = 0;
- hash_add(ctx->mmu_hash, &pgt_info->node, addr);
+ hash_add(ctx->mmu_shadow_hash, &pgt_info->node, shadow_addr);
+
+ return shadow_addr;
+
+shadow_err:
+ gen_pool_free(hdev->mmu_pgt_pool, phys_addr, prop->mmu_hop_table_size);
+pool_add_err:
+ kfree(pgt_info);
+
+ return ULLONG_MAX;
+}
+
+static inline u64 get_phys_hop0_addr(struct hl_ctx *ctx)
+{
+ return ctx->hdev->asic_prop.mmu_pgt_addr +
+ (ctx->asid * ctx->hdev->asic_prop.mmu_hop_table_size);
+}
+
+static inline u64 get_hop0_addr(struct hl_ctx *ctx)
+{
+ return (u64) (uintptr_t) ctx->hdev->mmu_shadow_hop0 +
+ (ctx->asid * ctx->hdev->asic_prop.mmu_hop_table_size);
+}
+
+static inline void flush(struct hl_ctx *ctx)
+{
+ /* flush all writes from all cores to reach PCI */
+ mb();
+ ctx->hdev->asic_funcs->read_pte(ctx->hdev, get_phys_hop0_addr(ctx));
+}
+
+/* transform the value to physical address when writing to H/W */
+static inline void write_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val)
+{
+ /*
+ * The value to write is actually the address of the next shadow hop +
+ * flags at the 12 LSBs.
+ * Hence in order to get the value to write to the physical PTE, we
+ * clear the 12 LSBs and translate the shadow hop to its associated
+ * physical hop, and add back the original 12 LSBs.
+ */
+ u64 phys_val = get_phys_addr(ctx, val & PTE_PHYS_ADDR_MASK) |
+ (val & OFFSET_MASK);
+
+ ctx->hdev->asic_funcs->write_pte(ctx->hdev,
+ get_phys_addr(ctx, shadow_pte_addr),
+ phys_val);
+
+ *(u64 *) (uintptr_t) shadow_pte_addr = val;
+}
- return addr;
+/* do not transform the value to physical address when writing to H/W */
+static inline void write_final_pte(struct hl_ctx *ctx, u64 shadow_pte_addr,
+ u64 val)
+{
+ ctx->hdev->asic_funcs->write_pte(ctx->hdev,
+ get_phys_addr(ctx, shadow_pte_addr),
+ val);
+ *(u64 *) (uintptr_t) shadow_pte_addr = val;
}
-static inline void clear_pte(struct hl_device *hdev, u64 pte_addr)
+/* clear the last and present bits */
+static inline void clear_pte(struct hl_ctx *ctx, u64 pte_addr)
{
- /* clear the last and present bits */
- hdev->asic_funcs->write_pte(hdev, pte_addr, 0);
+ /* no need to transform the value to physical address */
+ write_final_pte(ctx, pte_addr, 0);
}
static inline void get_pte(struct hl_ctx *ctx, u64 hop_addr)
@@ -98,12 +164,6 @@ static inline int put_pte(struct hl_ctx *ctx, u64 hop_addr)
return num_of_ptes_left;
}
-static inline u64 get_hop0_addr(struct hl_ctx *ctx)
-{
- return ctx->hdev->asic_prop.mmu_pgt_addr +
- (ctx->asid * ctx->hdev->asic_prop.mmu_hop_table_size);
-}
-
static inline u64 get_hopN_pte_addr(struct hl_ctx *ctx, u64 hop_addr,
u64 virt_addr, u64 mask, u64 shift)
{
@@ -136,7 +196,7 @@ static inline u64 get_hop4_pte_addr(struct hl_ctx *ctx, u64 hop_addr, u64 vaddr)
return get_hopN_pte_addr(ctx, hop_addr, vaddr, HOP4_MASK, HOP4_SHIFT);
}
-static inline u64 get_next_hop_addr(u64 curr_pte)
+static inline u64 get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte)
{
if (curr_pte & PAGE_PRESENT_MASK)
return curr_pte & PHYS_ADDR_MASK;
@@ -147,7 +207,7 @@ static inline u64 get_next_hop_addr(u64 curr_pte)
static inline u64 get_alloc_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte,
bool *is_new_hop)
{
- u64 hop_addr = get_next_hop_addr(curr_pte);
+ u64 hop_addr = get_next_hop_addr(ctx, curr_pte);
if (hop_addr == ULLONG_MAX) {
hop_addr = alloc_hop(ctx);
@@ -157,106 +217,30 @@ static inline u64 get_alloc_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte,
return hop_addr;
}
-/*
- * hl_mmu_init - init the mmu module
- *
- * @hdev: pointer to the habanalabs device structure
- *
- * This function does the following:
- * - Allocate max_asid zeroed hop0 pgts so no mapping is available
- * - Enable mmu in hw
- * - Invalidate the mmu cache
- * - Create a pool of pages for pgts
- * - Returns 0 on success
- *
- * This function depends on DMA QMAN to be working!
- */
-int hl_mmu_init(struct hl_device *hdev)
+/* translates shadow address inside hop to a physical address */
+static inline u64 get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr)
{
- struct asic_fixed_properties *prop = &hdev->asic_prop;
- int rc;
+ u64 page_mask = (ctx->hdev->asic_prop.mmu_hop_table_size - 1);
+ u64 shadow_hop_addr = shadow_addr & ~page_mask;
+ u64 pte_offset = shadow_addr & page_mask;
+ u64 phys_hop_addr;
- if (!hdev->mmu_enable)
- return 0;
-
- /* MMU HW init was already done in device hw_init() */
-
- mutex_init(&hdev->mmu_cache_lock);
-
- hdev->mmu_pgt_pool =
- gen_pool_create(__ffs(prop->mmu_hop_table_size), -1);
-
- if (!hdev->mmu_pgt_pool) {
- dev_err(hdev->dev, "Failed to create page gen pool\n");
- rc = -ENOMEM;
- goto err_pool_create;
- }
-
- rc = gen_pool_add(hdev->mmu_pgt_pool, prop->mmu_pgt_addr +
- prop->mmu_hop0_tables_total_size,
- prop->mmu_pgt_size - prop->mmu_hop0_tables_total_size,
- -1);
- if (rc) {
- dev_err(hdev->dev, "Failed to add memory to page gen pool\n");
- goto err_pool_add;
- }
-
- return 0;
-
-err_pool_add:
- gen_pool_destroy(hdev->mmu_pgt_pool);
-err_pool_create:
- mutex_destroy(&hdev->mmu_cache_lock);
+ if (shadow_hop_addr != get_hop0_addr(ctx))
+ phys_hop_addr = get_pgt_info(ctx, shadow_hop_addr)->phys_addr;
+ else
+ phys_hop_addr = get_phys_hop0_addr(ctx);
- return rc;
+ return phys_hop_addr + pte_offset;
}
-/*
- * hl_mmu_fini - release the mmu module.
- *
- * @hdev: pointer to the habanalabs device structure
- *
- * This function does the following:
- * - Disable mmu in hw
- * - free the pgts pool
- *
- * All ctxs should be freed before calling this func
- */
-void hl_mmu_fini(struct hl_device *hdev)
-{
- if (!hdev->mmu_enable)
- return;
-
- gen_pool_destroy(hdev->mmu_pgt_pool);
-
- mutex_destroy(&hdev->mmu_cache_lock);
-
- /* MMU HW fini will be done in device hw_fini() */
-}
-
-/**
- * hl_mmu_ctx_init() - initialize a context for using the MMU module.
- * @ctx: pointer to the context structure to initialize.
- *
- * Initialize a mutex to protect the concurrent mapping flow, a hash to hold all
- * page tables hops related to this context and an optional DRAM default page
- * mapping.
- * Return: 0 on success, non-zero otherwise.
- */
-int hl_mmu_ctx_init(struct hl_ctx *ctx)
+static int dram_default_mapping_init(struct hl_ctx *ctx)
{
struct hl_device *hdev = ctx->hdev;
struct asic_fixed_properties *prop = &hdev->asic_prop;
- u64 num_of_hop3, total_hops, hop1_addr, hop2_addr, hop2_pte_addr,
- hop3_pte_addr, pte_val;
+ u64 num_of_hop3, total_hops, hop0_addr, hop1_addr, hop2_addr,
+ hop2_pte_addr, hop3_pte_addr, pte_val;
int rc, i, j, hop3_allocated = 0;
- if (!hdev->mmu_enable)
- return 0;
-
- mutex_init(&ctx->mmu_lock);
- hash_init(ctx->mmu_hash);
-
if (!hdev->dram_supports_virtual_memory ||
!hdev->dram_default_page_mapping)
return 0;
@@ -269,10 +253,10 @@ int hl_mmu_ctx_init(struct hl_ctx *ctx)
total_hops = num_of_hop3 + 2;
ctx->dram_default_hops = kzalloc(HL_PTE_SIZE * total_hops, GFP_KERNEL);
- if (!ctx->dram_default_hops) {
- rc = -ENOMEM;
- goto alloc_err;
- }
+ if (!ctx->dram_default_hops)
+ return -ENOMEM;
+
+ hop0_addr = get_hop0_addr(ctx);
hop1_addr = alloc_hop(ctx);
if (hop1_addr == ULLONG_MAX) {
@@ -304,17 +288,17 @@ int hl_mmu_ctx_init(struct hl_ctx *ctx)
/* need only pte 0 in hops 0 and 1 */
pte_val = (hop1_addr & PTE_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
- hdev->asic_funcs->write_pte(hdev, get_hop0_addr(ctx), pte_val);
+ write_pte(ctx, hop0_addr, pte_val);
pte_val = (hop2_addr & PTE_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
- hdev->asic_funcs->write_pte(hdev, hop1_addr, pte_val);
+ write_pte(ctx, hop1_addr, pte_val);
get_pte(ctx, hop1_addr);
hop2_pte_addr = hop2_addr;
for (i = 0 ; i < num_of_hop3 ; i++) {
pte_val = (ctx->dram_default_hops[i] & PTE_PHYS_ADDR_MASK) |
PAGE_PRESENT_MASK;
- hdev->asic_funcs->write_pte(hdev, hop2_pte_addr, pte_val);
+ write_pte(ctx, hop2_pte_addr, pte_val);
get_pte(ctx, hop2_addr);
hop2_pte_addr += HL_PTE_SIZE;
}
@@ -325,33 +309,183 @@ int hl_mmu_ctx_init(struct hl_ctx *ctx)
for (i = 0 ; i < num_of_hop3 ; i++) {
hop3_pte_addr = ctx->dram_default_hops[i];
for (j = 0 ; j < PTE_ENTRIES_IN_HOP ; j++) {
- hdev->asic_funcs->write_pte(hdev, hop3_pte_addr,
- pte_val);
+ write_final_pte(ctx, hop3_pte_addr, pte_val);
get_pte(ctx, ctx->dram_default_hops[i]);
hop3_pte_addr += HL_PTE_SIZE;
}
}
- /* flush all writes to reach PCI */
- mb();
- hdev->asic_funcs->read_pte(hdev, hop2_addr);
+ flush(ctx);
return 0;
hop3_err:
for (i = 0 ; i < hop3_allocated ; i++)
free_hop(ctx, ctx->dram_default_hops[i]);
+
free_hop(ctx, hop2_addr);
hop2_err:
free_hop(ctx, hop1_addr);
hop1_err:
kfree(ctx->dram_default_hops);
-alloc_err:
- mutex_destroy(&ctx->mmu_lock);
return rc;
}
+static void dram_default_mapping_fini(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u64 num_of_hop3, total_hops, hop0_addr, hop1_addr, hop2_addr,
+ hop2_pte_addr, hop3_pte_addr;
+ int i, j;
+
+ if (!hdev->dram_supports_virtual_memory ||
+ !hdev->dram_default_page_mapping)
+ return;
+
+ num_of_hop3 = prop->dram_size_for_default_page_mapping;
+ do_div(num_of_hop3, prop->dram_page_size);
+ do_div(num_of_hop3, PTE_ENTRIES_IN_HOP);
+
+ hop0_addr = get_hop0_addr(ctx);
+ /* add hop1 and hop2 */
+ total_hops = num_of_hop3 + 2;
+ hop1_addr = ctx->dram_default_hops[total_hops - 1];
+ hop2_addr = ctx->dram_default_hops[total_hops - 2];
+
+ for (i = 0 ; i < num_of_hop3 ; i++) {
+ hop3_pte_addr = ctx->dram_default_hops[i];
+ for (j = 0 ; j < PTE_ENTRIES_IN_HOP ; j++) {
+ clear_pte(ctx, hop3_pte_addr);
+ put_pte(ctx, ctx->dram_default_hops[i]);
+ hop3_pte_addr += HL_PTE_SIZE;
+ }
+ }
+
+ hop2_pte_addr = hop2_addr;
+ hop2_pte_addr = hop2_addr;
+ for (i = 0 ; i < num_of_hop3 ; i++) {
+ clear_pte(ctx, hop2_pte_addr);
+ put_pte(ctx, hop2_addr);
+ hop2_pte_addr += HL_PTE_SIZE;
+ }
+
+ clear_pte(ctx, hop1_addr);
+ put_pte(ctx, hop1_addr);
+ clear_pte(ctx, hop0_addr);
+
+ kfree(ctx->dram_default_hops);
+
+ flush(ctx);
+}
+
+/**
+ * hl_mmu_init() - initialize the MMU module.
+ * @hdev: habanalabs device structure.
+ *
+ * This function does the following:
+ * - Allocate max_asid zeroed hop0 pgts so no mapping is available.
+ * - Enable MMU in H/W.
+ * - Invalidate the MMU cache.
+ * - Create a pool of pages for pgt_infos.
+ *
+ * This function depends on DMA QMAN to be working!
+ *
+ * Return: 0 for success, non-zero for failure.
+ */
+int hl_mmu_init(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ int rc;
+
+ if (!hdev->mmu_enable)
+ return 0;
+
+ /* MMU H/W init was already done in device hw_init() */
+
+ mutex_init(&hdev->mmu_cache_lock);
+
+ hdev->mmu_pgt_pool =
+ gen_pool_create(__ffs(prop->mmu_hop_table_size), -1);
+
+ if (!hdev->mmu_pgt_pool) {
+ dev_err(hdev->dev, "Failed to create page gen pool\n");
+ rc = -ENOMEM;
+ goto err_pool_create;
+ }
+
+ rc = gen_pool_add(hdev->mmu_pgt_pool, prop->mmu_pgt_addr +
+ prop->mmu_hop0_tables_total_size,
+ prop->mmu_pgt_size - prop->mmu_hop0_tables_total_size,
+ -1);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to add memory to page gen pool\n");
+ goto err_pool_add;
+ }
+
+ hdev->mmu_shadow_hop0 = kvmalloc_array(prop->max_asid,
+ prop->mmu_hop_table_size,
+ GFP_KERNEL | __GFP_ZERO);
+ if (!hdev->mmu_shadow_hop0) {
+ rc = -ENOMEM;
+ goto err_pool_add;
+ }
+
+ return 0;
+
+err_pool_add:
+ gen_pool_destroy(hdev->mmu_pgt_pool);
+err_pool_create:
+ mutex_destroy(&hdev->mmu_cache_lock);
+
+ return rc;
+}
+
+/**
+ * hl_mmu_fini() - release the MMU module.
+ * @hdev: habanalabs device structure.
+ *
+ * This function does the following:
+ * - Disable MMU in H/W.
+ * - Free the pgt_infos pool.
+ *
+ * All contexts should be freed before calling this function.
+ */
+void hl_mmu_fini(struct hl_device *hdev)
+{
+ if (!hdev->mmu_enable)
+ return;
+
+ kvfree(hdev->mmu_shadow_hop0);
+ gen_pool_destroy(hdev->mmu_pgt_pool);
+ mutex_destroy(&hdev->mmu_cache_lock);
+
+ /* MMU H/W fini will be done in device hw_fini() */
+}
+
+/**
+ * hl_mmu_ctx_init() - initialize a context for using the MMU module.
+ * @ctx: pointer to the context structure to initialize.
+ *
+ * Initialize a mutex to protect the concurrent mapping flow, a hash to hold all
+ * page tables hops related to this context.
+ * Return: 0 on success, non-zero otherwise.
+ */
+int hl_mmu_ctx_init(struct hl_ctx *ctx)
+{
+ struct hl_device *hdev = ctx->hdev;
+
+ if (!hdev->mmu_enable)
+ return 0;
+
+ mutex_init(&ctx->mmu_lock);
+ hash_init(ctx->mmu_phys_hash);
+ hash_init(ctx->mmu_shadow_hash);
+
+ return dram_default_mapping_init(ctx);
+}
+
/*
* hl_mmu_ctx_fini - disable a ctx from using the mmu module
*
@@ -365,63 +499,23 @@ alloc_err:
void hl_mmu_ctx_fini(struct hl_ctx *ctx)
{
struct hl_device *hdev = ctx->hdev;
- struct asic_fixed_properties *prop = &hdev->asic_prop;
struct pgt_info *pgt_info;
struct hlist_node *tmp;
- u64 num_of_hop3, total_hops, hop1_addr, hop2_addr, hop2_pte_addr,
- hop3_pte_addr;
- int i, j;
+ int i;
- if (!ctx->hdev->mmu_enable)
+ if (!hdev->mmu_enable)
return;
- if (hdev->dram_supports_virtual_memory &&
- hdev->dram_default_page_mapping) {
-
- num_of_hop3 = prop->dram_size_for_default_page_mapping;
- do_div(num_of_hop3, prop->dram_page_size);
- do_div(num_of_hop3, PTE_ENTRIES_IN_HOP);
-
- /* add hop1 and hop2 */
- total_hops = num_of_hop3 + 2;
- hop1_addr = ctx->dram_default_hops[total_hops - 1];
- hop2_addr = ctx->dram_default_hops[total_hops - 2];
-
- for (i = 0 ; i < num_of_hop3 ; i++) {
- hop3_pte_addr = ctx->dram_default_hops[i];
- for (j = 0 ; j < PTE_ENTRIES_IN_HOP ; j++) {
- clear_pte(hdev, hop3_pte_addr);
- put_pte(ctx, ctx->dram_default_hops[i]);
- hop3_pte_addr += HL_PTE_SIZE;
- }
- }
+ dram_default_mapping_fini(ctx);
- hop2_pte_addr = hop2_addr;
- for (i = 0 ; i < num_of_hop3 ; i++) {
- clear_pte(hdev, hop2_pte_addr);
- put_pte(ctx, hop2_addr);
- hop2_pte_addr += HL_PTE_SIZE;
- }
-
- clear_pte(hdev, hop1_addr);
- put_pte(ctx, hop1_addr);
- clear_pte(hdev, get_hop0_addr(ctx));
-
- kfree(ctx->dram_default_hops);
-
- /* flush all writes to reach PCI */
- mb();
- hdev->asic_funcs->read_pte(hdev, hop2_addr);
- }
-
- if (!hash_empty(ctx->mmu_hash))
+ if (!hash_empty(ctx->mmu_shadow_hash))
dev_err(hdev->dev, "ctx is freed while it has pgts in use\n");
- hash_for_each_safe(ctx->mmu_hash, i, tmp, pgt_info, node) {
+ hash_for_each_safe(ctx->mmu_shadow_hash, i, tmp, pgt_info, node) {
dev_err(hdev->dev,
"pgt_info of addr 0x%llx of asid %d was not destroyed, num_ptes: %d\n",
- pgt_info->addr, ctx->asid, pgt_info->num_of_ptes);
- free_hop(ctx, pgt_info->addr);
+ pgt_info->phys_addr, ctx->asid, pgt_info->num_of_ptes);
+ free_hop(ctx, pgt_info->shadow_addr);
}
mutex_destroy(&ctx->mmu_lock);
@@ -437,45 +531,43 @@ static int _hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr)
hop3_addr = 0, hop3_pte_addr = 0,
hop4_addr = 0, hop4_pte_addr = 0,
curr_pte;
- int clear_hop3 = 1;
- bool is_dram_addr, is_huge, is_dram_default_page_mapping;
+ bool is_dram_addr, is_huge, clear_hop3 = true;
is_dram_addr = hl_mem_area_inside_range(virt_addr, PAGE_SIZE_2MB,
prop->va_space_dram_start_address,
prop->va_space_dram_end_address);
hop0_addr = get_hop0_addr(ctx);
-
hop0_pte_addr = get_hop0_pte_addr(ctx, hop0_addr, virt_addr);
- curr_pte = hdev->asic_funcs->read_pte(hdev, hop0_pte_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop0_pte_addr;
- hop1_addr = get_next_hop_addr(curr_pte);
+ hop1_addr = get_next_hop_addr(ctx, curr_pte);
if (hop1_addr == ULLONG_MAX)
goto not_mapped;
hop1_pte_addr = get_hop1_pte_addr(ctx, hop1_addr, virt_addr);
- curr_pte = hdev->asic_funcs->read_pte(hdev, hop1_pte_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop1_pte_addr;
- hop2_addr = get_next_hop_addr(curr_pte);
+ hop2_addr = get_next_hop_addr(ctx, curr_pte);
if (hop2_addr == ULLONG_MAX)
goto not_mapped;
hop2_pte_addr = get_hop2_pte_addr(ctx, hop2_addr, virt_addr);
- curr_pte = hdev->asic_funcs->read_pte(hdev, hop2_pte_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop2_pte_addr;
- hop3_addr = get_next_hop_addr(curr_pte);
+ hop3_addr = get_next_hop_addr(ctx, curr_pte);
if (hop3_addr == ULLONG_MAX)
goto not_mapped;
hop3_pte_addr = get_hop3_pte_addr(ctx, hop3_addr, virt_addr);
- curr_pte = hdev->asic_funcs->read_pte(hdev, hop3_pte_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop3_pte_addr;
is_huge = curr_pte & LAST_MASK;
@@ -485,27 +577,24 @@ static int _hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr)
return -EFAULT;
}
- is_dram_default_page_mapping =
- hdev->dram_default_page_mapping && is_dram_addr;
-
if (!is_huge) {
- hop4_addr = get_next_hop_addr(curr_pte);
+ hop4_addr = get_next_hop_addr(ctx, curr_pte);
if (hop4_addr == ULLONG_MAX)
goto not_mapped;
hop4_pte_addr = get_hop4_pte_addr(ctx, hop4_addr, virt_addr);
- curr_pte = hdev->asic_funcs->read_pte(hdev, hop4_pte_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop4_pte_addr;
- clear_hop3 = 0;
+ clear_hop3 = false;
}
- if (is_dram_default_page_mapping) {
- u64 zero_pte = (prop->mmu_dram_default_page_addr &
+ if (hdev->dram_default_page_mapping && is_dram_addr) {
+ u64 default_pte = (prop->mmu_dram_default_page_addr &
PTE_PHYS_ADDR_MASK) | LAST_MASK |
PAGE_PRESENT_MASK;
- if (curr_pte == zero_pte) {
+ if (curr_pte == default_pte) {
dev_err(hdev->dev,
"DRAM: hop3 PTE points to zero page, can't unmap, va: 0x%llx\n",
virt_addr);
@@ -519,40 +608,43 @@ static int _hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr)
goto not_mapped;
}
- hdev->asic_funcs->write_pte(hdev, hop3_pte_addr, zero_pte);
+ write_final_pte(ctx, hop3_pte_addr, default_pte);
put_pte(ctx, hop3_addr);
} else {
if (!(curr_pte & PAGE_PRESENT_MASK))
goto not_mapped;
- clear_pte(hdev, hop4_addr ? hop4_pte_addr : hop3_pte_addr);
+ if (hop4_addr)
+ clear_pte(ctx, hop4_pte_addr);
+ else
+ clear_pte(ctx, hop3_pte_addr);
if (hop4_addr && !put_pte(ctx, hop4_addr))
- clear_hop3 = 1;
+ clear_hop3 = true;
if (!clear_hop3)
goto flush;
- clear_pte(hdev, hop3_pte_addr);
+
+ clear_pte(ctx, hop3_pte_addr);
if (put_pte(ctx, hop3_addr))
goto flush;
- clear_pte(hdev, hop2_pte_addr);
+
+ clear_pte(ctx, hop2_pte_addr);
if (put_pte(ctx, hop2_addr))
goto flush;
- clear_pte(hdev, hop1_pte_addr);
+
+ clear_pte(ctx, hop1_pte_addr);
if (put_pte(ctx, hop1_addr))
goto flush;
- clear_pte(hdev, hop0_pte_addr);
+
+ clear_pte(ctx, hop0_pte_addr);
}
flush:
- /* flush all writes from all cores to reach PCI */
- mb();
-
- hdev->asic_funcs->read_pte(hdev,
- hop4_addr ? hop4_pte_addr : hop3_pte_addr);
+ flush(ctx);
return 0;
@@ -632,8 +724,7 @@ static int _hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
hop4_addr = 0, hop4_pte_addr = 0,
curr_pte = 0;
bool hop1_new = false, hop2_new = false, hop3_new = false,
- hop4_new = false, is_huge, is_dram_addr,
- is_dram_default_page_mapping;
+ hop4_new = false, is_huge, is_dram_addr;
int rc = -ENOMEM;
/*
@@ -654,59 +745,46 @@ static int _hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
return -EFAULT;
}
- is_dram_default_page_mapping =
- hdev->dram_default_page_mapping && is_dram_addr;
-
hop0_addr = get_hop0_addr(ctx);
-
hop0_pte_addr = get_hop0_pte_addr(ctx, hop0_addr, virt_addr);
-
- curr_pte = hdev->asic_funcs->read_pte(hdev, hop0_pte_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop0_pte_addr;
hop1_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop1_new);
-
if (hop1_addr == ULLONG_MAX)
goto err;
hop1_pte_addr = get_hop1_pte_addr(ctx, hop1_addr, virt_addr);
-
- curr_pte = hdev->asic_funcs->read_pte(hdev, hop1_pte_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop1_pte_addr;
hop2_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop2_new);
-
if (hop2_addr == ULLONG_MAX)
goto err;
hop2_pte_addr = get_hop2_pte_addr(ctx, hop2_addr, virt_addr);
-
- curr_pte = hdev->asic_funcs->read_pte(hdev, hop2_pte_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop2_pte_addr;
hop3_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop3_new);
-
if (hop3_addr == ULLONG_MAX)
goto err;
hop3_pte_addr = get_hop3_pte_addr(ctx, hop3_addr, virt_addr);
-
- curr_pte = hdev->asic_funcs->read_pte(hdev, hop3_pte_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop3_pte_addr;
if (!is_huge) {
hop4_addr = get_alloc_next_hop_addr(ctx, curr_pte, &hop4_new);
-
if (hop4_addr == ULLONG_MAX)
goto err;
hop4_pte_addr = get_hop4_pte_addr(ctx, hop4_addr, virt_addr);
-
- curr_pte = hdev->asic_funcs->read_pte(hdev, hop4_pte_addr);
+ curr_pte = *(u64 *) (uintptr_t) hop4_pte_addr;
}
- if (is_dram_default_page_mapping) {
- u64 zero_pte = (prop->mmu_dram_default_page_addr &
+ if (hdev->dram_default_page_mapping && is_dram_addr) {
+ u64 default_pte = (prop->mmu_dram_default_page_addr &
PTE_PHYS_ADDR_MASK) | LAST_MASK |
PAGE_PRESENT_MASK;
- if (curr_pte != zero_pte) {
+ if (curr_pte != default_pte) {
dev_err(hdev->dev,
"DRAM: mapping already exists for virt_addr 0x%llx\n",
virt_addr);
@@ -722,27 +800,22 @@ static int _hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
}
} else if (curr_pte & PAGE_PRESENT_MASK) {
dev_err(hdev->dev,
- "mapping already exists for virt_addr 0x%llx\n",
- virt_addr);
+ "mapping already exists for virt_addr 0x%llx\n",
+ virt_addr);
dev_dbg(hdev->dev, "hop0 pte: 0x%llx (0x%llx)\n",
- hdev->asic_funcs->read_pte(hdev, hop0_pte_addr),
- hop0_pte_addr);
+ *(u64 *) (uintptr_t) hop0_pte_addr, hop0_pte_addr);
dev_dbg(hdev->dev, "hop1 pte: 0x%llx (0x%llx)\n",
- hdev->asic_funcs->read_pte(hdev, hop1_pte_addr),
- hop1_pte_addr);
+ *(u64 *) (uintptr_t) hop1_pte_addr, hop1_pte_addr);
dev_dbg(hdev->dev, "hop2 pte: 0x%llx (0x%llx)\n",
- hdev->asic_funcs->read_pte(hdev, hop2_pte_addr),
- hop2_pte_addr);
+ *(u64 *) (uintptr_t) hop2_pte_addr, hop2_pte_addr);
dev_dbg(hdev->dev, "hop3 pte: 0x%llx (0x%llx)\n",
- hdev->asic_funcs->read_pte(hdev, hop3_pte_addr),
- hop3_pte_addr);
+ *(u64 *) (uintptr_t) hop3_pte_addr, hop3_pte_addr);
if (!is_huge)
dev_dbg(hdev->dev, "hop4 pte: 0x%llx (0x%llx)\n",
- hdev->asic_funcs->read_pte(hdev,
- hop4_pte_addr),
- hop4_pte_addr);
+ *(u64 *) (uintptr_t) hop4_pte_addr,
+ hop4_pte_addr);
rc = -EINVAL;
goto err;
@@ -751,28 +824,26 @@ static int _hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
curr_pte = (phys_addr & PTE_PHYS_ADDR_MASK) | LAST_MASK
| PAGE_PRESENT_MASK;
- hdev->asic_funcs->write_pte(hdev,
- is_huge ? hop3_pte_addr : hop4_pte_addr,
- curr_pte);
+ if (is_huge)
+ write_final_pte(ctx, hop3_pte_addr, curr_pte);
+ else
+ write_final_pte(ctx, hop4_pte_addr, curr_pte);
if (hop1_new) {
- curr_pte = (hop1_addr & PTE_PHYS_ADDR_MASK) |
- PAGE_PRESENT_MASK;
- ctx->hdev->asic_funcs->write_pte(ctx->hdev, hop0_pte_addr,
- curr_pte);
+ curr_pte =
+ (hop1_addr & PTE_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
+ write_pte(ctx, hop0_pte_addr, curr_pte);
}
if (hop2_new) {
- curr_pte = (hop2_addr & PTE_PHYS_ADDR_MASK) |
- PAGE_PRESENT_MASK;
- ctx->hdev->asic_funcs->write_pte(ctx->hdev, hop1_pte_addr,
- curr_pte);
+ curr_pte =
+ (hop2_addr & PTE_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
+ write_pte(ctx, hop1_pte_addr, curr_pte);
get_pte(ctx, hop1_addr);
}
if (hop3_new) {
- curr_pte = (hop3_addr & PTE_PHYS_ADDR_MASK) |
- PAGE_PRESENT_MASK;
- ctx->hdev->asic_funcs->write_pte(ctx->hdev, hop2_pte_addr,
- curr_pte);
+ curr_pte =
+ (hop3_addr & PTE_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;
+ write_pte(ctx, hop2_pte_addr, curr_pte);
get_pte(ctx, hop2_addr);
}
@@ -780,8 +851,7 @@ static int _hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
if (hop4_new) {
curr_pte = (hop4_addr & PTE_PHYS_ADDR_MASK) |
PAGE_PRESENT_MASK;
- ctx->hdev->asic_funcs->write_pte(ctx->hdev,
- hop3_pte_addr, curr_pte);
+ write_pte(ctx, hop3_pte_addr, curr_pte);
get_pte(ctx, hop3_addr);
}
@@ -790,11 +860,7 @@ static int _hl_mmu_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
get_pte(ctx, hop3_addr);
}
- /* flush all writes from all cores to reach PCI */
- mb();
-
- hdev->asic_funcs->read_pte(hdev,
- is_huge ? hop3_pte_addr : hop4_pte_addr);
+ flush(ctx);
return 0;
diff --git a/drivers/misc/habanalabs/pci.c b/drivers/misc/habanalabs/pci.c
new file mode 100644
index 000000000000..0e78a04d63f4
--- /dev/null
+++ b/drivers/misc/habanalabs/pci.c
@@ -0,0 +1,408 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2019 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#include "habanalabs.h"
+#include "include/hw_ip/pci/pci_general.h"
+
+#include <linux/pci.h>
+
+/**
+ * hl_pci_bars_map() - Map PCI BARs.
+ * @hdev: Pointer to hl_device structure.
+ * @bar_name: Array of BAR names.
+ * @is_wc: Array with flag per BAR whether a write-combined mapping is needed.
+ *
+ * Request PCI regions and map them to kernel virtual addresses.
+ *
+ * Return: 0 on success, non-zero for failure.
+ */
+int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
+ bool is_wc[3])
+{
+ struct pci_dev *pdev = hdev->pdev;
+ int rc, i, bar;
+
+ rc = pci_request_regions(pdev, HL_NAME);
+ if (rc) {
+ dev_err(hdev->dev, "Cannot obtain PCI resources\n");
+ return rc;
+ }
+
+ for (i = 0 ; i < 3 ; i++) {
+ bar = i * 2; /* 64-bit BARs */
+ hdev->pcie_bar[bar] = is_wc[i] ?
+ pci_ioremap_wc_bar(pdev, bar) :
+ pci_ioremap_bar(pdev, bar);
+ if (!hdev->pcie_bar[bar]) {
+ dev_err(hdev->dev, "pci_ioremap%s_bar failed for %s\n",
+ is_wc[i] ? "_wc" : "", name[i]);
+ rc = -ENODEV;
+ goto err;
+ }
+ }
+
+ return 0;
+
+err:
+ for (i = 2 ; i >= 0 ; i--) {
+ bar = i * 2; /* 64-bit BARs */
+ if (hdev->pcie_bar[bar])
+ iounmap(hdev->pcie_bar[bar]);
+ }
+
+ pci_release_regions(pdev);
+
+ return rc;
+}
+
+/*
+ * hl_pci_bars_unmap() - Unmap PCI BARS.
+ * @hdev: Pointer to hl_device structure.
+ *
+ * Release all PCI BARs and unmap their virtual addresses.
+ */
+static void hl_pci_bars_unmap(struct hl_device *hdev)
+{
+ struct pci_dev *pdev = hdev->pdev;
+ int i, bar;
+
+ for (i = 2 ; i >= 0 ; i--) {
+ bar = i * 2; /* 64-bit BARs */
+ iounmap(hdev->pcie_bar[bar]);
+ }
+
+ pci_release_regions(pdev);
+}
+
+/*
+ * hl_pci_elbi_write() - Write through the ELBI interface.
+ * @hdev: Pointer to hl_device structure.
+ *
+ * Return: 0 on success, negative value for failure.
+ */
+static int hl_pci_elbi_write(struct hl_device *hdev, u64 addr, u32 data)
+{
+ struct pci_dev *pdev = hdev->pdev;
+ ktime_t timeout;
+ u32 val;
+
+ /* Clear previous status */
+ pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, 0);
+
+ pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_ADDR, (u32) addr);
+ pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_DATA, data);
+ pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_CTRL,
+ PCI_CONFIG_ELBI_CTRL_WRITE);
+
+ timeout = ktime_add_ms(ktime_get(), 10);
+ for (;;) {
+ pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, &val);
+ if (val & PCI_CONFIG_ELBI_STS_MASK)
+ break;
+ if (ktime_compare(ktime_get(), timeout) > 0) {
+ pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS,
+ &val);
+ break;
+ }
+
+ usleep_range(300, 500);
+ }
+
+ if ((val & PCI_CONFIG_ELBI_STS_MASK) == PCI_CONFIG_ELBI_STS_DONE)
+ return 0;
+
+ if (val & PCI_CONFIG_ELBI_STS_ERR) {
+ dev_err(hdev->dev, "Error writing to ELBI\n");
+ return -EIO;
+ }
+
+ if (!(val & PCI_CONFIG_ELBI_STS_MASK)) {
+ dev_err(hdev->dev, "ELBI write didn't finish in time\n");
+ return -EIO;
+ }
+
+ dev_err(hdev->dev, "ELBI write has undefined bits in status\n");
+ return -EIO;
+}
+
+/**
+ * hl_pci_iatu_write() - iatu write routine.
+ * @hdev: Pointer to hl_device structure.
+ *
+ * Return: 0 on success, negative value for failure.
+ */
+int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u32 dbi_offset;
+ int rc;
+
+ dbi_offset = addr & 0xFFF;
+
+ rc = hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0x00300000);
+ rc |= hl_pci_elbi_write(hdev, prop->pcie_dbi_base_address + dbi_offset,
+ data);
+
+ if (rc)
+ return -EIO;
+
+ return 0;
+}
+
+/*
+ * hl_pci_reset_link_through_bridge() - Reset PCI link.
+ * @hdev: Pointer to hl_device structure.
+ */
+static void hl_pci_reset_link_through_bridge(struct hl_device *hdev)
+{
+ struct pci_dev *pdev = hdev->pdev;
+ struct pci_dev *parent_port;
+ u16 val;
+
+ parent_port = pdev->bus->self;
+ pci_read_config_word(parent_port, PCI_BRIDGE_CONTROL, &val);
+ val |= PCI_BRIDGE_CTL_BUS_RESET;
+ pci_write_config_word(parent_port, PCI_BRIDGE_CONTROL, val);
+ ssleep(1);
+
+ val &= ~(PCI_BRIDGE_CTL_BUS_RESET);
+ pci_write_config_word(parent_port, PCI_BRIDGE_CONTROL, val);
+ ssleep(3);
+}
+
+/**
+ * hl_pci_set_dram_bar_base() - Set DDR BAR to map specific device address.
+ * @hdev: Pointer to hl_device structure.
+ * @inbound_region: Inbound region number.
+ * @bar: PCI BAR number.
+ * @addr: Address in DRAM. Must be aligned to DRAM bar size.
+ *
+ * Configure the iATU so that the DRAM bar will start at the specified address.
+ *
+ * Return: 0 on success, negative value for failure.
+ */
+int hl_pci_set_dram_bar_base(struct hl_device *hdev, u8 inbound_region, u8 bar,
+ u64 addr)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u32 offset;
+ int rc;
+
+ switch (inbound_region) {
+ case 0:
+ offset = 0x100;
+ break;
+ case 1:
+ offset = 0x300;
+ break;
+ case 2:
+ offset = 0x500;
+ break;
+ default:
+ dev_err(hdev->dev, "Invalid inbound region %d\n",
+ inbound_region);
+ return -EINVAL;
+ }
+
+ if (bar != 0 && bar != 2 && bar != 4) {
+ dev_err(hdev->dev, "Invalid PCI BAR %d\n", bar);
+ return -EINVAL;
+ }
+
+ /* Point to the specified address */
+ rc = hl_pci_iatu_write(hdev, offset + 0x14, lower_32_bits(addr));
+ rc |= hl_pci_iatu_write(hdev, offset + 0x18, upper_32_bits(addr));
+ rc |= hl_pci_iatu_write(hdev, offset + 0x0, 0);
+ /* Enable + BAR match + match enable + BAR number */
+ rc |= hl_pci_iatu_write(hdev, offset + 0x4, 0xC0080000 | (bar << 8));
+
+ /* Return the DBI window to the default location */
+ rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0);
+ rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr + 4, 0);
+
+ if (rc)
+ dev_err(hdev->dev, "failed to map DRAM bar to 0x%08llx\n",
+ addr);
+
+ return rc;
+}
+
+/**
+ * hl_pci_init_iatu() - Initialize the iATU unit inside the PCI controller.
+ * @hdev: Pointer to hl_device structure.
+ * @sram_base_address: SRAM base address.
+ * @dram_base_address: DRAM base address.
+ * @host_phys_base_address: Base physical address of host memory for device
+ * transactions.
+ * @host_phys_size: Size of host memory for device transactions.
+ *
+ * This is needed in case the firmware doesn't initialize the iATU.
+ *
+ * Return: 0 on success, negative value for failure.
+ */
+int hl_pci_init_iatu(struct hl_device *hdev, u64 sram_base_address,
+ u64 dram_base_address, u64 host_phys_base_address,
+ u64 host_phys_size)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u64 host_phys_end_addr;
+ int rc = 0;
+
+ /* Inbound Region 0 - Bar 0 - Point to SRAM base address */
+ rc = hl_pci_iatu_write(hdev, 0x114, lower_32_bits(sram_base_address));
+ rc |= hl_pci_iatu_write(hdev, 0x118, upper_32_bits(sram_base_address));
+ rc |= hl_pci_iatu_write(hdev, 0x100, 0);
+ /* Enable + Bar match + match enable */
+ rc |= hl_pci_iatu_write(hdev, 0x104, 0xC0080000);
+
+ /* Point to DRAM */
+ if (!hdev->asic_funcs->set_dram_bar_base)
+ return -EINVAL;
+ if (hdev->asic_funcs->set_dram_bar_base(hdev, dram_base_address) ==
+ U64_MAX)
+ return -EIO;
+
+
+ /* Outbound Region 0 - Point to Host */
+ host_phys_end_addr = host_phys_base_address + host_phys_size - 1;
+ rc |= hl_pci_iatu_write(hdev, 0x008,
+ lower_32_bits(host_phys_base_address));
+ rc |= hl_pci_iatu_write(hdev, 0x00C,
+ upper_32_bits(host_phys_base_address));
+ rc |= hl_pci_iatu_write(hdev, 0x010, lower_32_bits(host_phys_end_addr));
+ rc |= hl_pci_iatu_write(hdev, 0x014, 0);
+ rc |= hl_pci_iatu_write(hdev, 0x018, 0);
+ rc |= hl_pci_iatu_write(hdev, 0x020, upper_32_bits(host_phys_end_addr));
+ /* Increase region size */
+ rc |= hl_pci_iatu_write(hdev, 0x000, 0x00002000);
+ /* Enable */
+ rc |= hl_pci_iatu_write(hdev, 0x004, 0x80000000);
+
+ /* Return the DBI window to the default location */
+ rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0);
+ rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr + 4, 0);
+
+ if (rc)
+ return -EIO;
+
+ return 0;
+}
+
+/**
+ * hl_pci_set_dma_mask() - Set DMA masks for the device.
+ * @hdev: Pointer to hl_device structure.
+ * @dma_mask: number of bits for the requested dma mask.
+ *
+ * This function sets the DMA masks (regular and consistent) for a specified
+ * value. If it doesn't succeed, it tries to set it to a fall-back value
+ *
+ * Return: 0 on success, non-zero for failure.
+ */
+int hl_pci_set_dma_mask(struct hl_device *hdev, u8 dma_mask)
+{
+ struct pci_dev *pdev = hdev->pdev;
+ int rc;
+
+ /* set DMA mask */
+ rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_mask));
+ if (rc) {
+ dev_warn(hdev->dev,
+ "Failed to set pci dma mask to %d bits, error %d\n",
+ dma_mask, rc);
+
+ dma_mask = hdev->dma_mask;
+
+ rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_mask));
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to set pci dma mask to %d bits, error %d\n",
+ dma_mask, rc);
+ return rc;
+ }
+ }
+
+ /*
+ * We managed to set the dma mask, so update the dma mask field. If
+ * the set to the coherent mask will fail with that mask, we will
+ * fail the entire function
+ */
+ hdev->dma_mask = dma_mask;
+
+ rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(dma_mask));
+ if (rc) {
+ dev_err(hdev->dev,
+ "Failed to set pci consistent dma mask to %d bits, error %d\n",
+ dma_mask, rc);
+ return rc;
+ }
+
+ return 0;
+}
+
+/**
+ * hl_pci_init() - PCI initialization code.
+ * @hdev: Pointer to hl_device structure.
+ * @dma_mask: number of bits for the requested dma mask.
+ *
+ * Set DMA masks, initialize the PCI controller and map the PCI BARs.
+ *
+ * Return: 0 on success, non-zero for failure.
+ */
+int hl_pci_init(struct hl_device *hdev, u8 dma_mask)
+{
+ struct pci_dev *pdev = hdev->pdev;
+ int rc;
+
+ rc = hl_pci_set_dma_mask(hdev, dma_mask);
+ if (rc)
+ return rc;
+
+ if (hdev->reset_pcilink)
+ hl_pci_reset_link_through_bridge(hdev);
+
+ rc = pci_enable_device_mem(pdev);
+ if (rc) {
+ dev_err(hdev->dev, "can't enable PCI device\n");
+ return rc;
+ }
+
+ pci_set_master(pdev);
+
+ rc = hdev->asic_funcs->init_iatu(hdev);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to initialize iATU\n");
+ goto disable_device;
+ }
+
+ rc = hdev->asic_funcs->pci_bars_map(hdev);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to initialize PCI BARs\n");
+ goto disable_device;
+ }
+
+ return 0;
+
+disable_device:
+ pci_clear_master(pdev);
+ pci_disable_device(pdev);
+
+ return rc;
+}
+
+/**
+ * hl_fw_fini() - PCI finalization code.
+ * @hdev: Pointer to hl_device structure
+ *
+ * Unmap PCI bars and disable PCI device.
+ */
+void hl_pci_fini(struct hl_device *hdev)
+{
+ hl_pci_bars_unmap(hdev);
+
+ pci_clear_master(hdev->pdev);
+ pci_disable_device(hdev->pdev);
+}
diff --git a/drivers/misc/kgdbts.c b/drivers/misc/kgdbts.c
index de20bdaa148d..8b01257783dd 100644
--- a/drivers/misc/kgdbts.c
+++ b/drivers/misc/kgdbts.c
@@ -1135,7 +1135,7 @@ static void kgdbts_put_char(u8 chr)
static int param_set_kgdbts_var(const char *kmessage,
const struct kernel_param *kp)
{
- int len = strlen(kmessage);
+ size_t len = strlen(kmessage);
if (len >= MAX_CONFIG_LEN) {
printk(KERN_ERR "kgdbts: config string too long\n");
@@ -1155,7 +1155,7 @@ static int param_set_kgdbts_var(const char *kmessage,
strcpy(config, kmessage);
/* Chop out \n char as a result of echo */
- if (config[len - 1] == '\n')
+ if (len && config[len - 1] == '\n')
config[len - 1] = '\0';
/* Go and configure with the new params. */
diff --git a/drivers/misc/mei/Kconfig b/drivers/misc/mei/Kconfig
index 74e2c667dce0..9d7b3719bfa0 100644
--- a/drivers/misc/mei/Kconfig
+++ b/drivers/misc/mei/Kconfig
@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2003-2019, Intel Corporation. All rights reserved.
config INTEL_MEI
tristate "Intel Management Engine Interface"
depends on X86 && PCI
@@ -44,12 +46,4 @@ config INTEL_MEI_TXE
Supported SoCs:
Intel Bay Trail
-config INTEL_MEI_HDCP
- tristate "Intel HDCP2.2 services of ME Interface"
- select INTEL_MEI_ME
- depends on DRM_I915
- help
- MEI Support for HDCP2.2 Services on Intel platforms.
-
- Enables the ME FW services required for HDCP2.2 support through
- I915 display driver of Intel.
+source "drivers/misc/mei/hdcp/Kconfig"
diff --git a/drivers/misc/mei/Makefile b/drivers/misc/mei/Makefile
index 8c2d9565a4cb..f1c76f7ee804 100644
--- a/drivers/misc/mei/Makefile
+++ b/drivers/misc/mei/Makefile
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
#
+# Copyright (c) 2010-2019, Intel Corporation. All rights reserved.
# Makefile - Intel Management Engine Interface (Intel MEI) Linux driver
-# Copyright (c) 2010-2014, Intel Corporation.
#
obj-$(CONFIG_INTEL_MEI) += mei.o
mei-objs := init.o
diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c
index 5fcac02233af..32e9b1aed2ca 100644
--- a/drivers/misc/mei/bus-fixup.c
+++ b/drivers/misc/mei/bus-fixup.c
@@ -1,17 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2013-2019, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2018, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#include <linux/kernel.h>
diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c
index 65bec998eb6e..985bd4fd3328 100644
--- a/drivers/misc/mei/bus.c
+++ b/drivers/misc/mei/bus.c
@@ -1,16 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
+ * Copyright (c) 2012-2019, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2012-2013, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#include <linux/module.h>
diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c
index ca4c9cc218a2..1e3edbbacb1e 100644
--- a/drivers/misc/mei/client.c
+++ b/drivers/misc/mei/client.c
@@ -1,17 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2003-2019, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2012, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#include <linux/sched/signal.h>
@@ -679,7 +669,7 @@ int mei_cl_unlink(struct mei_cl *cl)
void mei_host_client_init(struct mei_device *dev)
{
- dev->dev_state = MEI_DEV_ENABLED;
+ mei_set_devstate(dev, MEI_DEV_ENABLED);
dev->reset_count = 0;
schedule_work(&dev->bus_rescan_work);
diff --git a/drivers/misc/mei/client.h b/drivers/misc/mei/client.h
index 64e318f589b4..c1f9e810cf81 100644
--- a/drivers/misc/mei/client.h
+++ b/drivers/misc/mei/client.h
@@ -1,17 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
- *
+ * Copyright (c) 2003-2018, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2012, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#ifndef _MEI_CLIENT_H_
diff --git a/drivers/misc/mei/debugfs.c b/drivers/misc/mei/debugfs.c
index 7b5df8fd6c5a..0970142bcace 100644
--- a/drivers/misc/mei/debugfs.c
+++ b/drivers/misc/mei/debugfs.c
@@ -1,18 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2012-2016, Intel Corporation. All rights reserved
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2012-2013, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
+
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/device.h>
diff --git a/drivers/misc/mei/dma-ring.c b/drivers/misc/mei/dma-ring.c
index 795641b82181..ef56f849b251 100644
--- a/drivers/misc/mei/dma-ring.c
+++ b/drivers/misc/mei/dma-ring.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
/*
- * Copyright(c) 2016 - 2018 Intel Corporation. All rights reserved.
+ * Copyright(c) 2016-2018 Intel Corporation. All rights reserved.
*/
#include <linux/dma-mapping.h>
#include <linux/mei.h>
diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c
index e6207f614816..a44094cdbc36 100644
--- a/drivers/misc/mei/hbm.c
+++ b/drivers/misc/mei/hbm.c
@@ -1,19 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2003-2019, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2012, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
-
#include <linux/export.h>
#include <linux/sched.h>
#include <linux/wait.h>
diff --git a/drivers/misc/mei/hbm.h b/drivers/misc/mei/hbm.h
index 0171a7e79bab..5aa58cffdd2e 100644
--- a/drivers/misc/mei/hbm.h
+++ b/drivers/misc/mei/hbm.h
@@ -1,17 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
- *
+ * Copyright (c) 2003-2018, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2012, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#ifndef _MEI_HBM_H_
diff --git a/drivers/misc/mei/hdcp/Kconfig b/drivers/misc/mei/hdcp/Kconfig
new file mode 100644
index 000000000000..95b2d6d37f10
--- /dev/null
+++ b/drivers/misc/mei/hdcp/Kconfig
@@ -0,0 +1,13 @@
+
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2019, Intel Corporation. All rights reserved.
+#
+config INTEL_MEI_HDCP
+ tristate "Intel HDCP2.2 services of ME Interface"
+ select INTEL_MEI_ME
+ depends on DRM_I915
+ help
+ MEI Support for HDCP2.2 Services on Intel platforms.
+
+ Enables the ME FW services required for HDCP2.2 support through
+ I915 display driver of Intel.
diff --git a/drivers/misc/mei/hdcp/Makefile b/drivers/misc/mei/hdcp/Makefile
index adbe7506282d..3fbb56485ce8 100644
--- a/drivers/misc/mei/hdcp/Makefile
+++ b/drivers/misc/mei/hdcp/Makefile
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0
#
-# Copyright (c) 2019, Intel Corporation.
+# Copyright (c) 2019, Intel Corporation. All rights reserved.
#
# Makefile - HDCP client driver for Intel MEI Bus Driver.
diff --git a/drivers/misc/mei/hdcp/mei_hdcp.c b/drivers/misc/mei/hdcp/mei_hdcp.c
index 90b6ae8e9dae..b07000202d4a 100644
--- a/drivers/misc/mei/hdcp/mei_hdcp.c
+++ b/drivers/misc/mei/hdcp/mei_hdcp.c
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: (GPL-2.0)
+// SPDX-License-Identifier: GPL-2.0
/*
* Copyright © 2019 Intel Corporation
*
diff --git a/drivers/misc/mei/hdcp/mei_hdcp.h b/drivers/misc/mei/hdcp/mei_hdcp.h
index 5f74b908e486..e4b1cd54c853 100644
--- a/drivers/misc/mei/hdcp/mei_hdcp.h
+++ b/drivers/misc/mei/hdcp/mei_hdcp.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0+) */
+/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright © 2019 Intel Corporation
*
diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h
index bb1ee9834a02..d74b182e19f3 100644
--- a/drivers/misc/mei/hw-me-regs.h
+++ b/drivers/misc/mei/hw-me-regs.h
@@ -1,68 +1,8 @@
-/******************************************************************************
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (c) 2003-2019, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Intel MEI Interface Header
- *
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2003 - 2012 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
- * The full GNU General Public License is included in this distribution
- * in the file called LICENSE.GPL.
- *
- * Contact Information:
- * Intel Corporation.
- * linux-mei@linux.intel.com
- * http://www.intel.com
- *
- * BSD LICENSE
- *
- * Copyright(c) 2003 - 2012 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- *****************************************************************************/
+ */
#ifndef _MEI_HW_MEI_REGS_H_
#define _MEI_HW_MEI_REGS_H_
diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c
index 8a47a6fc3fc7..abe1b1f4362f 100644
--- a/drivers/misc/mei/hw-me.c
+++ b/drivers/misc/mei/hw-me.c
@@ -1,17 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2003-2018, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2012, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#include <linux/pci.h>
diff --git a/drivers/misc/mei/hw-me.h b/drivers/misc/mei/hw-me.h
index bbcc5fc106cd..08c84a0de4a8 100644
--- a/drivers/misc/mei/hw-me.h
+++ b/drivers/misc/mei/hw-me.h
@@ -1,21 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
- *
+ * Copyright (c) 2012-2018, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2012, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
-
-
#ifndef _MEI_INTERFACE_H_
#define _MEI_INTERFACE_H_
diff --git a/drivers/misc/mei/hw-txe-regs.h b/drivers/misc/mei/hw-txe-regs.h
index f19229c4e655..a92b306dac8b 100644
--- a/drivers/misc/mei/hw-txe-regs.h
+++ b/drivers/misc/mei/hw-txe-regs.h
@@ -1,63 +1,8 @@
-/******************************************************************************
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (c) 2013-2014, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Intel MEI Interface Header
- *
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2013 - 2014 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- *
- * The full GNU General Public License is included in this distribution
- * in the file called COPYING
- *
- * Contact Information:
- * Intel Corporation.
- * linux-mei@linux.intel.com
- * http://www.intel.com
- *
- * BSD LICENSE
- *
- * Copyright(c) 2013 - 2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- *****************************************************************************/
+ */
#ifndef _MEI_HW_TXE_REGS_H_
#define _MEI_HW_TXE_REGS_H_
diff --git a/drivers/misc/mei/hw-txe.c b/drivers/misc/mei/hw-txe.c
index 8449fe0367ff..5e58656b8e19 100644
--- a/drivers/misc/mei/hw-txe.c
+++ b/drivers/misc/mei/hw-txe.c
@@ -1,17 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2013-2014, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2013-2014, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#include <linux/pci.h>
diff --git a/drivers/misc/mei/hw-txe.h b/drivers/misc/mei/hw-txe.h
index e1e8b66d7648..96511b04bf88 100644
--- a/drivers/misc/mei/hw-txe.h
+++ b/drivers/misc/mei/hw-txe.h
@@ -1,17 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
- *
+ * Copyright (c) 2013-2016, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2013-2014, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#ifndef _MEI_HW_TXE_H_
diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h
index b7d2487b8409..d025a5f8317e 100644
--- a/drivers/misc/mei/hw.h
+++ b/drivers/misc/mei/hw.h
@@ -1,17 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
- *
+ * Copyright (c) 2003-2018, Intel Corporation. All rights reserved
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2012, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#ifndef _MEI_HW_TYPES_H_
diff --git a/drivers/misc/mei/init.c b/drivers/misc/mei/init.c
index eb026e2a0537..b9fef773e71b 100644
--- a/drivers/misc/mei/init.c
+++ b/drivers/misc/mei/init.c
@@ -1,17 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2012-2018, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2012, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#include <linux/export.h>
@@ -133,12 +123,12 @@ int mei_reset(struct mei_device *dev)
/* enter reset flow */
interrupts_enabled = state != MEI_DEV_POWER_DOWN;
- dev->dev_state = MEI_DEV_RESETTING;
+ mei_set_devstate(dev, MEI_DEV_RESETTING);
dev->reset_count++;
if (dev->reset_count > MEI_MAX_CONSEC_RESET) {
dev_err(dev->dev, "reset: reached maximal consecutive resets: disabling the device\n");
- dev->dev_state = MEI_DEV_DISABLED;
+ mei_set_devstate(dev, MEI_DEV_DISABLED);
return -ENODEV;
}
@@ -160,7 +150,7 @@ int mei_reset(struct mei_device *dev)
if (state == MEI_DEV_POWER_DOWN) {
dev_dbg(dev->dev, "powering down: end of reset\n");
- dev->dev_state = MEI_DEV_DISABLED;
+ mei_set_devstate(dev, MEI_DEV_DISABLED);
return 0;
}
@@ -172,11 +162,11 @@ int mei_reset(struct mei_device *dev)
dev_dbg(dev->dev, "link is established start sending messages.\n");
- dev->dev_state = MEI_DEV_INIT_CLIENTS;
+ mei_set_devstate(dev, MEI_DEV_INIT_CLIENTS);
ret = mei_hbm_start_req(dev);
if (ret) {
dev_err(dev->dev, "hbm_start failed ret = %d\n", ret);
- dev->dev_state = MEI_DEV_RESETTING;
+ mei_set_devstate(dev, MEI_DEV_RESETTING);
return ret;
}
@@ -206,7 +196,7 @@ int mei_start(struct mei_device *dev)
dev->reset_count = 0;
do {
- dev->dev_state = MEI_DEV_INITIALIZING;
+ mei_set_devstate(dev, MEI_DEV_INITIALIZING);
ret = mei_reset(dev);
if (ret == -ENODEV || dev->dev_state == MEI_DEV_DISABLED) {
@@ -241,7 +231,7 @@ int mei_start(struct mei_device *dev)
return 0;
err:
dev_err(dev->dev, "link layer initialization failed.\n");
- dev->dev_state = MEI_DEV_DISABLED;
+ mei_set_devstate(dev, MEI_DEV_DISABLED);
mutex_unlock(&dev->device_lock);
return -ENODEV;
}
@@ -260,7 +250,7 @@ int mei_restart(struct mei_device *dev)
mutex_lock(&dev->device_lock);
- dev->dev_state = MEI_DEV_POWER_UP;
+ mei_set_devstate(dev, MEI_DEV_POWER_UP);
dev->reset_count = 0;
err = mei_reset(dev);
@@ -311,7 +301,7 @@ void mei_stop(struct mei_device *dev)
dev_dbg(dev->dev, "stopping the device.\n");
mutex_lock(&dev->device_lock);
- dev->dev_state = MEI_DEV_POWER_DOWN;
+ mei_set_devstate(dev, MEI_DEV_POWER_DOWN);
mutex_unlock(&dev->device_lock);
mei_cl_bus_remove_devices(dev);
@@ -324,7 +314,7 @@ void mei_stop(struct mei_device *dev)
mei_reset(dev);
/* move device to disabled state unconditionally */
- dev->dev_state = MEI_DEV_DISABLED;
+ mei_set_devstate(dev, MEI_DEV_DISABLED);
mutex_unlock(&dev->device_lock);
}
diff --git a/drivers/misc/mei/interrupt.c b/drivers/misc/mei/interrupt.c
index 055c2d89b310..c70a8c74cc57 100644
--- a/drivers/misc/mei/interrupt.c
+++ b/drivers/misc/mei/interrupt.c
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2003-2018, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2012, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
-
#include <linux/export.h>
#include <linux/kthread.h>
#include <linux/interrupt.h>
diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c
index 87281b3695e6..ad02097d7fee 100644
--- a/drivers/misc/mei/main.c
+++ b/drivers/misc/mei/main.c
@@ -1,18 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2003-2018, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2018, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
+
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/kernel.h>
@@ -37,6 +28,12 @@
#include "mei_dev.h"
#include "client.h"
+static struct class *mei_class;
+static dev_t mei_devt;
+#define MEI_MAX_DEVS MINORMASK
+static DEFINE_MUTEX(mei_minor_lock);
+static DEFINE_IDR(mei_idr);
+
/**
* mei_open - the open function
*
@@ -838,12 +835,65 @@ static ssize_t fw_ver_show(struct device *device,
}
static DEVICE_ATTR_RO(fw_ver);
+/**
+ * dev_state_show - display device state
+ *
+ * @device: device pointer
+ * @attr: attribute pointer
+ * @buf: char out buffer
+ *
+ * Return: number of the bytes printed into buf or error
+ */
+static ssize_t dev_state_show(struct device *device,
+ struct device_attribute *attr, char *buf)
+{
+ struct mei_device *dev = dev_get_drvdata(device);
+ enum mei_dev_state dev_state;
+
+ mutex_lock(&dev->device_lock);
+ dev_state = dev->dev_state;
+ mutex_unlock(&dev->device_lock);
+
+ return sprintf(buf, "%s", mei_dev_state_str(dev_state));
+}
+static DEVICE_ATTR_RO(dev_state);
+
+static int match_devt(struct device *dev, const void *data)
+{
+ const dev_t *devt = data;
+
+ return dev->devt == *devt;
+}
+
+/**
+ * dev_set_devstate: set to new device state and notify sysfs file.
+ *
+ * @dev: mei_device
+ * @state: new device state
+ */
+void mei_set_devstate(struct mei_device *dev, enum mei_dev_state state)
+{
+ struct device *clsdev;
+
+ if (dev->dev_state == state)
+ return;
+
+ dev->dev_state = state;
+
+ clsdev = class_find_device(mei_class, NULL, &dev->cdev.dev, match_devt);
+ if (clsdev) {
+ sysfs_notify(&clsdev->kobj, NULL, "dev_state");
+ put_device(clsdev);
+ }
+}
+
static struct attribute *mei_attrs[] = {
&dev_attr_fw_status.attr,
&dev_attr_hbm_ver.attr,
&dev_attr_hbm_ver_drv.attr,
&dev_attr_tx_queue_limit.attr,
&dev_attr_fw_ver.attr,
+ &dev_attr_dev_state.attr,
NULL
};
ATTRIBUTE_GROUPS(mei);
@@ -867,12 +917,6 @@ static const struct file_operations mei_fops = {
.llseek = no_llseek
};
-static struct class *mei_class;
-static dev_t mei_devt;
-#define MEI_MAX_DEVS MINORMASK
-static DEFINE_MUTEX(mei_minor_lock);
-static DEFINE_IDR(mei_idr);
-
/**
* mei_minor_get - obtain next free device minor number
*
diff --git a/drivers/misc/mei/mei-trace.c b/drivers/misc/mei/mei-trace.c
index 374edde72a14..48d4c4fcefd2 100644
--- a/drivers/misc/mei/mei-trace.c
+++ b/drivers/misc/mei/mei-trace.c
@@ -1,17 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2015-2016, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#include <linux/module.h>
diff --git a/drivers/misc/mei/mei-trace.h b/drivers/misc/mei/mei-trace.h
index b52e9b97a7c0..df758033dc93 100644
--- a/drivers/misc/mei/mei-trace.h
+++ b/drivers/misc/mei/mei-trace.h
@@ -1,17 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
- *
+ * Copyright (c) 2015-2016, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#if !defined(_MEI_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
diff --git a/drivers/misc/mei/mei_dev.h b/drivers/misc/mei/mei_dev.h
index 685b78ce30a5..fca832fcac57 100644
--- a/drivers/misc/mei/mei_dev.h
+++ b/drivers/misc/mei/mei_dev.h
@@ -1,17 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
- *
+ * Copyright (c) 2003-2018, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2018, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#ifndef _MEI_DEV_H_
@@ -535,7 +525,6 @@ struct mei_device {
struct dentry *dbgfs_dir;
#endif /* CONFIG_DEBUG_FS */
-
const struct mei_hw_ops *ops;
char hw[0] __aligned(sizeof(void *));
};
@@ -594,6 +583,8 @@ int mei_restart(struct mei_device *dev);
void mei_stop(struct mei_device *dev);
void mei_cancel_work(struct mei_device *dev);
+void mei_set_devstate(struct mei_device *dev, enum mei_dev_state state);
+
int mei_dmam_ring_alloc(struct mei_device *dev);
void mei_dmam_ring_free(struct mei_device *dev);
bool mei_dma_ring_is_allocated(struct mei_device *dev);
diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c
index 3ab946ad3257..7a2b3545a7f9 100644
--- a/drivers/misc/mei/pci-me.c
+++ b/drivers/misc/mei/pci-me.c
@@ -1,18 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2003-2019, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2003-2012, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
+
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/kernel.h>
diff --git a/drivers/misc/mei/pci-txe.c b/drivers/misc/mei/pci-txe.c
index e1b909123fb0..2e37fc2e0fa8 100644
--- a/drivers/misc/mei/pci-txe.c
+++ b/drivers/misc/mei/pci-txe.c
@@ -1,17 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- *
+ * Copyright (c) 2013-2017, Intel Corporation. All rights reserved.
* Intel Management Engine Interface (Intel MEI) Linux driver
- * Copyright (c) 2013-2014, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
*/
#include <linux/module.h>
diff --git a/drivers/misc/sgi-xp/xpc_uv.c b/drivers/misc/sgi-xp/xpc_uv.c
index 9e443df44b3b..0c6de97dd347 100644
--- a/drivers/misc/sgi-xp/xpc_uv.c
+++ b/drivers/misc/sgi-xp/xpc_uv.c
@@ -572,6 +572,7 @@ xpc_handle_activate_mq_msg_uv(struct xpc_partition *part,
xpc_wakeup_channel_mgr(part);
}
+ /* fall through */
case XPC_ACTIVATE_MQ_MSG_MARK_ENGAGED_UV:
spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
part_uv->flags |= XPC_P_ENGAGED_UV;