// SPDX-License-Identifier: GPL-2.0 /* * Copyright 2016-2022 HabanaLabs, Ltd. * All Rights Reserved. */ #define pr_fmt(fmt) "habanalabs: " fmt #include #include "habanalabs.h" #include #include #include #include #define HL_RESET_DELAY_USEC 10000 /* 10ms */ #define HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC 5 enum dma_alloc_type { DMA_ALLOC_COHERENT, DMA_ALLOC_CPU_ACCESSIBLE, DMA_ALLOC_POOL, }; #define MEM_SCRUB_DEFAULT_VAL 0x1122334455667788 /* * hl_set_dram_bar- sets the bar to allow later access to address * * @hdev: pointer to habanalabs device structure. * @addr: the address the caller wants to access. * @region: the PCI region. * @new_bar_region_base: the new BAR region base address. * * @return: the old BAR base address on success, U64_MAX for failure. * The caller should set it back to the old address after use. * * In case the bar space does not cover the whole address space, * the bar base address should be set to allow access to a given address. * This function can be called also if the bar doesn't need to be set, * in that case it just won't change the base. */ static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region, u64 *new_bar_region_base) { struct asic_fixed_properties *prop = &hdev->asic_prop; u64 bar_base_addr, old_base; if (is_power_of_2(prop->dram_pci_bar_size)) bar_base_addr = addr & ~(prop->dram_pci_bar_size - 0x1ull); else bar_base_addr = DIV_ROUND_DOWN_ULL(addr, prop->dram_pci_bar_size) * prop->dram_pci_bar_size; old_base = hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr); /* in case of success we need to update the new BAR base */ if ((old_base != U64_MAX) && new_bar_region_base) *new_bar_region_base = bar_base_addr; return old_base; } int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val, enum debugfs_access_type acc_type, enum pci_region region_type, bool set_dram_bar) { struct pci_mem_region *region = &hdev->pci_mem_region[region_type]; u64 old_base = 0, rc, bar_region_base = region->region_base; void __iomem *acc_addr; if (set_dram_bar) { old_base = hl_set_dram_bar(hdev, addr, region, &bar_region_base); if (old_base == U64_MAX) return -EIO; } acc_addr = hdev->pcie_bar[region->bar_id] + region->offset_in_bar + (addr - bar_region_base); switch (acc_type) { case DEBUGFS_READ8: *val = readb(acc_addr); break; case DEBUGFS_WRITE8: writeb(*val, acc_addr); break; case DEBUGFS_READ32: *val = readl(acc_addr); break; case DEBUGFS_WRITE32: writel(*val, acc_addr); break; case DEBUGFS_READ64: *val = readq(acc_addr); break; case DEBUGFS_WRITE64: writeq(*val, acc_addr); break; } if (set_dram_bar) { rc = hl_set_dram_bar(hdev, old_base, region, NULL); if (rc == U64_MAX) return -EIO; } return 0; } static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, gfp_t flag, enum dma_alloc_type alloc_type, const char *caller) { void *ptr = NULL; switch (alloc_type) { case DMA_ALLOC_COHERENT: ptr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, size, dma_handle, flag); break; case DMA_ALLOC_CPU_ACCESSIBLE: ptr = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); break; case DMA_ALLOC_POOL: ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, size, flag, dma_handle); break; } if (trace_habanalabs_dma_alloc_enabled() && !ZERO_OR_NULL_PTR(ptr)) trace_habanalabs_dma_alloc(hdev->dev, (u64) (uintptr_t) ptr, *dma_handle, size, caller); return ptr; } static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr, dma_addr_t dma_handle, enum dma_alloc_type alloc_type, const char *caller) { /* this is needed to avoid warning on using freed pointer */ u64 store_cpu_addr = (u64) (uintptr_t) cpu_addr; switch (alloc_type) { case DMA_ALLOC_COHERENT: hdev->asic_funcs->asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle); break; case DMA_ALLOC_CPU_ACCESSIBLE: hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, cpu_addr); break; case DMA_ALLOC_POOL: hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle); break; } trace_habanalabs_dma_free(hdev->dev, store_cpu_addr, dma_handle, size, caller); } void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, gfp_t flag, const char *caller) { return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT, caller); } void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr, dma_addr_t dma_handle, const char *caller) { hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller); } void *hl_cpu_accessible_dma_pool_alloc_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, const char *caller) { return hl_dma_alloc_common(hdev, size, dma_handle, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller); } void hl_cpu_accessible_dma_pool_free_caller(struct hl_device *hdev, size_t size, void *vaddr, const char *caller) { hl_asic_dma_free_common(hdev, size, vaddr, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller); } void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags, dma_addr_t *dma_handle, const char *caller) { return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL, caller); } void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr, const char *caller) { hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller); } int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct scatterlist *sg; int rc, i; rc = dma_map_sgtable(&hdev->pdev->dev, sgt, dir, 0); if (rc) return rc; /* Shift to the device's base physical address of host memory if necessary */ if (prop->device_dma_offset_for_host_access) for_each_sgtable_dma_sg(sgt, sg, i) sg->dma_address += prop->device_dma_offset_for_host_access; return 0; } void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct scatterlist *sg; int i; /* Cancel the device's base physical address of host memory if necessary */ if (prop->device_dma_offset_for_host_access) for_each_sgtable_dma_sg(sgt, sg, i) sg->dma_address -= prop->device_dma_offset_for_host_access; dma_unmap_sgtable(&hdev->pdev->dev, sgt, dir, 0); } /* * hl_access_cfg_region - access the config region * * @hdev: pointer to habanalabs device structure * @addr: the address to access * @val: the value to write from or read to * @acc_type: the type of access (read/write 64/32) */ int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val, enum debugfs_access_type acc_type) { struct pci_mem_region *cfg_region = &hdev->pci_mem_region[PCI_REGION_CFG]; u32 val_h, val_l; if (!IS_ALIGNED(addr, sizeof(u32))) { dev_err(hdev->dev, "address %#llx not a multiple of %zu\n", addr, sizeof(u32)); return -EINVAL; } switch (acc_type) { case DEBUGFS_READ32: *val = RREG32(addr - cfg_region->region_base); break; case DEBUGFS_WRITE32: WREG32(addr - cfg_region->region_base, *val); break; case DEBUGFS_READ64: val_l = RREG32(addr - cfg_region->region_base); val_h = RREG32(addr + sizeof(u32) - cfg_region->region_base); *val = (((u64) val_h) << 32) | val_l; break; case DEBUGFS_WRITE64: WREG32(addr - cfg_region->region_base, lower_32_bits(*val)); WREG32(addr + sizeof(u32) - cfg_region->region_base, upper_32_bits(*val)); break; default: dev_err(hdev->dev, "access type %d is not supported\n", acc_type); return -EOPNOTSUPP; } return 0; } /* * hl_access_dev_mem - access device memory * * @hdev: pointer to habanalabs device structure * @region_type: the type of the region the address belongs to * @addr: the address to access * @val: the value to write from or read to * @acc_type: the type of access (r/w, 32/64) */ int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type, u64 addr, u64 *val, enum debugfs_access_type acc_type) { switch (region_type) { case PCI_REGION_CFG: return hl_access_cfg_region(hdev, addr, val, acc_type); case PCI_REGION_SRAM: case PCI_REGION_DRAM: return hl_access_sram_dram_region(hdev, addr, val, acc_type, region_type, (region_type == PCI_REGION_DRAM)); default: return -EFAULT; } return 0; } void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...) { va_list args; int str_size; va_start(args, fmt); /* Calculate formatted string length. Assuming each string is null terminated, hence * increment result by 1 */ str_size = vsnprintf(NULL, 0, fmt, args) + 1; va_end(args); if ((e->actual_size + str_size) < e->allocated_buf_size) { va_start(args, fmt); vsnprintf(e->buf + e->actual_size, str_size, fmt, args); va_end(args); } /* Need to update the size even when not updating destination buffer to get the exact size * of all input strings */ e->actual_size += str_size; } enum hl_device_status hl_device_status(struct hl_device *hdev) { enum hl_device_status status; if (hdev->reset_info.in_reset) { if (hdev->reset_info.in_compute_reset) status = HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE; else status = HL_DEVICE_STATUS_IN_RESET; } else if (hdev->reset_info.needs_reset) { status = HL_DEVICE_STATUS_NEEDS_RESET; } else if (hdev->disabled) { status = HL_DEVICE_STATUS_MALFUNCTION; } else if (!hdev->init_done) { status = HL_DEVICE_STATUS_IN_DEVICE_CREATION; } else { status = HL_DEVICE_STATUS_OPERATIONAL; } return status; } bool hl_device_operational(struct hl_device *hdev, enum hl_device_status *status) { enum hl_device_status current_status; current_status = hl_device_status(hdev); if (status) *status = current_status; switch (current_status) { case HL_DEVICE_STATUS_IN_RESET: case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE: case HL_DEVICE_STATUS_MALFUNCTION: case HL_DEVICE_STATUS_NEEDS_RESET: return false; case HL_DEVICE_STATUS_OPERATIONAL: case HL_DEVICE_STATUS_IN_DEVICE_CREATION: default: return true; } } bool hl_ctrl_device_operational(struct hl_device *hdev, enum hl_device_status *status) { enum hl_device_status current_status; current_status = hl_device_status(hdev); if (status) *status = current_status; switch (current_status) { case HL_DEVICE_STATUS_MALFUNCTION: return false; case HL_DEVICE_STATUS_IN_RESET: case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE: case HL_DEVICE_STATUS_NEEDS_RESET: case HL_DEVICE_STATUS_OPERATIONAL: case HL_DEVICE_STATUS_IN_DEVICE_CREATION: default: return true; } } static void print_idle_status_mask(struct hl_device *hdev, const char *message, u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE]) { u32 pad_width[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {}; BUILD_BUG_ON(HL_BUSY_ENGINES_MASK_EXT_SIZE != 4); pad_width[3] = idle_mask[3] ? 16 : 0; pad_width[2] = idle_mask[2] || pad_width[3] ? 16 : 0; pad_width[1] = idle_mask[1] || pad_width[2] ? 16 : 0; pad_width[0] = idle_mask[0] || pad_width[1] ? 16 : 0; dev_err(hdev->dev, "%s (mask %0*llx_%0*llx_%0*llx_%0*llx)\n", message, pad_width[3], idle_mask[3], pad_width[2], idle_mask[2], pad_width[1], idle_mask[1], pad_width[0], idle_mask[0]); } static void hpriv_release(struct kref *ref) { u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; bool reset_device, device_is_idle = true; struct hl_fpriv *hpriv; struct hl_device *hdev; hpriv = container_of(ref, struct hl_fpriv, refcount); hdev = hpriv->hdev; hdev->asic_funcs->send_device_activity(hdev, false); put_pid(hpriv->taskpid); hl_debugfs_remove_file(hpriv); mutex_destroy(&hpriv->ctx_lock); mutex_destroy(&hpriv->restore_phase_mutex); /* Device should be reset if reset-upon-device-release is enabled, or if there is a pending * reset that waits for device release. */ reset_device = hdev->reset_upon_device_release || hdev->reset_info.watchdog_active; /* Unless device is reset in any case, check idle status and reset if device is not idle */ if (!reset_device && hdev->pdev && !hdev->pldm) device_is_idle = hdev->asic_funcs->is_device_idle(hdev, idle_mask, HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL); if (!device_is_idle) { print_idle_status_mask(hdev, "device is not idle after user context is closed", idle_mask); reset_device = true; } /* We need to remove the user from the list to make sure the reset process won't * try to kill the user process. Because, if we got here, it means there are no * more driver/device resources that the user process is occupying so there is * no need to kill it * * However, we can't set the compute_ctx to NULL at this stage. This is to prevent * a race between the release and opening the device again. We don't want to let * a user open the device while there a reset is about to happen. */ mutex_lock(&hdev->fpriv_list_lock); list_del(&hpriv->dev_node); mutex_unlock(&hdev->fpriv_list_lock); if (reset_device) { hl_device_reset(hdev, HL_DRV_RESET_DEV_RELEASE); } else { /* Scrubbing is handled within hl_device_reset(), so here need to do it directly */ int rc = hdev->asic_funcs->scrub_device_mem(hdev); if (rc) dev_err(hdev->dev, "failed to scrub memory from hpriv release (%d)\n", rc); } /* Now we can mark the compute_ctx as not active. Even if a reset is running in a different * thread, we don't care because the in_reset is marked so if a user will try to open * the device it will fail on that, even if compute_ctx is false. */ mutex_lock(&hdev->fpriv_list_lock); hdev->is_compute_ctx_active = false; mutex_unlock(&hdev->fpriv_list_lock); hdev->compute_ctx_in_release = 0; /* release the eventfd */ if (hpriv->notifier_event.eventfd) eventfd_ctx_put(hpriv->notifier_event.eventfd); mutex_destroy(&hpriv->notifier_event.lock); kfree(hpriv); } void hl_hpriv_get(struct hl_fpriv *hpriv) { kref_get(&hpriv->refcount); } int hl_hpriv_put(struct hl_fpriv *hpriv) { return kref_put(&hpriv->refcount, hpriv_release); } /* * hl_device_release - release function for habanalabs device * * @inode: pointer to inode structure * @filp: pointer to file structure * * Called when process closes an habanalabs device */ static int hl_device_release(struct inode *inode, struct file *filp) { struct hl_fpriv *hpriv = filp->private_data; struct hl_device *hdev = hpriv->hdev; filp->private_data = NULL; if (!hdev) { pr_crit("Closing FD after device was removed. Memory leak will occur and it is advised to reboot.\n"); put_pid(hpriv->taskpid); return 0; } /* Each pending user interrupt holds the user's context, hence we * must release them all before calling hl_ctx_mgr_fini(). */ hl_release_pending_user_interrupts(hpriv->hdev); hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr); hl_mem_mgr_fini(&hpriv->mem_mgr); hdev->compute_ctx_in_release = 1; if (!hl_hpriv_put(hpriv)) { dev_notice(hdev->dev, "User process closed FD but device still in use\n"); hl_device_reset(hdev, HL_DRV_RESET_HARD); } hdev->last_open_session_duration_jif = jiffies - hdev->last_successful_open_jif; return 0; } static int hl_device_release_ctrl(struct inode *inode, struct file *filp) { struct hl_fpriv *hpriv = filp->private_data; struct hl_device *hdev = hpriv->hdev; filp->private_data = NULL; if (!hdev) { pr_err("Closing FD after device was removed\n"); goto out; } mutex_lock(&hdev->fpriv_ctrl_list_lock); list_del(&hpriv->dev_node); mutex_unlock(&hdev->fpriv_ctrl_list_lock); out: /* release the eventfd */ if (hpriv->notifier_event.eventfd) eventfd_ctx_put(hpriv->notifier_event.eventfd); mutex_destroy(&hpriv->notifier_event.lock); put_pid(hpriv->taskpid); kfree(hpriv); return 0; } /* * hl_mmap - mmap function for habanalabs device * * @*filp: pointer to file structure * @*vma: pointer to vm_area_struct of the process * * Called when process does an mmap on habanalabs device. Call the relevant mmap * function at the end of the common code. */ static int hl_mmap(struct file *filp, struct vm_area_struct *vma) { struct hl_fpriv *hpriv = filp->private_data; struct hl_device *hdev = hpriv->hdev; unsigned long vm_pgoff; if (!hdev) { pr_err_ratelimited("Trying to mmap after device was removed! Please close FD\n"); return -ENODEV; } vm_pgoff = vma->vm_pgoff; switch (vm_pgoff & HL_MMAP_TYPE_MASK) { case HL_MMAP_TYPE_BLOCK: vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff); return hl_hw_block_mmap(hpriv, vma); case HL_MMAP_TYPE_CB: case HL_MMAP_TYPE_TS_BUFF: return hl_mem_mgr_mmap(&hpriv->mem_mgr, vma, NULL); } return -EINVAL; } static const struct file_operations hl_ops = { .owner = THIS_MODULE, .open = hl_device_open, .release = hl_device_release, .mmap = hl_mmap, .unlocked_ioctl = hl_ioctl, .compat_ioctl = hl_ioctl }; static const struct file_operations hl_ctrl_ops = { .owner = THIS_MODULE, .open = hl_device_open_ctrl, .release = hl_device_release_ctrl, .unlocked_ioctl = hl_ioctl_control, .compat_ioctl = hl_ioctl_control }; static void device_release_func(struct device *dev) { kfree(dev); } /* * device_init_cdev - Initialize cdev and device for habanalabs device * * @hdev: pointer to habanalabs device structure * @hclass: pointer to the class object of the device * @minor: minor number of the specific device * @fpos: file operations to install for this device * @name: name of the device as it will appear in the filesystem * @cdev: pointer to the char device object that will be initialized * @dev: pointer to the device object that will be initialized * * Initialize a cdev and a Linux device for habanalabs's device. */ static int device_init_cdev(struct hl_device *hdev, struct class *hclass, int minor, const struct file_operations *fops, char *name, struct cdev *cdev, struct device **dev) { cdev_init(cdev, fops); cdev->owner = THIS_MODULE; *dev = kzalloc(sizeof(**dev), GFP_KERNEL); if (!*dev) return -ENOMEM; device_initialize(*dev); (*dev)->devt = MKDEV(hdev->major, minor); (*dev)->class = hclass; (*dev)->release = device_release_func; dev_set_drvdata(*dev, hdev); dev_set_name(*dev, "%s", name); return 0; } static int device_cdev_sysfs_add(struct hl_device *hdev) { int rc; rc = cdev_device_add(&hdev->cdev, hdev->dev); if (rc) { dev_err(hdev->dev, "failed to add a char device to the system\n"); return rc; } rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl); if (rc) { dev_err(hdev->dev, "failed to add a control char device to the system\n"); goto delete_cdev_device; } /* hl_sysfs_init() must be done after adding the device to the system */ rc = hl_sysfs_init(hdev); if (rc) { dev_err(hdev->dev, "failed to initialize sysfs\n"); goto delete_ctrl_cdev_device; } hdev->cdev_sysfs_created = true; return 0; delete_ctrl_cdev_device: cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); delete_cdev_device: cdev_device_del(&hdev->cdev, hdev->dev); return rc; } static void device_cdev_sysfs_del(struct hl_device *hdev) { if (!hdev->cdev_sysfs_created) goto put_devices; hl_sysfs_fini(hdev); cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); cdev_device_del(&hdev->cdev, hdev->dev); put_devices: put_device(hdev->dev); put_device(hdev->dev_ctrl); } static void device_hard_reset_pending(struct work_struct *work) { struct hl_device_reset_work *device_reset_work = container_of(work, struct hl_device_reset_work, reset_work.work); struct hl_device *hdev = device_reset_work->hdev; u32 flags; int rc; flags = device_reset_work->flags | HL_DRV_RESET_FROM_RESET_THR; rc = hl_device_reset(hdev, flags); if ((rc == -EBUSY) && !hdev->device_fini_pending) { struct hl_ctx *ctx = hl_get_compute_ctx(hdev); if (ctx) { /* The read refcount value should subtracted by one, because the read is * protected with hl_get_compute_ctx(). */ dev_info(hdev->dev, "Could not reset device (compute_ctx refcount %u). will try again in %u seconds", kref_read(&ctx->refcount) - 1, HL_PENDING_RESET_PER_SEC); hl_ctx_put(ctx); } else { dev_info(hdev->dev, "Could not reset device. will try again in %u seconds", HL_PENDING_RESET_PER_SEC); } queue_delayed_work(hdev->reset_wq, &device_reset_work->reset_work, msecs_to_jiffies(HL_PENDING_RESET_PER_SEC * 1000)); } } static void device_release_watchdog_func(struct work_struct *work) { struct hl_device_reset_work *device_release_watchdog_work = container_of(work, struct hl_device_reset_work, reset_work.work); struct hl_device *hdev = device_release_watchdog_work->hdev; u32 flags; dev_dbg(hdev->dev, "Device wasn't released in time. Initiate device reset.\n"); flags = device_release_watchdog_work->flags | HL_DRV_RESET_FROM_WD_THR; hl_device_reset(hdev, flags); } /* * device_early_init - do some early initialization for the habanalabs device * * @hdev: pointer to habanalabs device structure * * Install the relevant function pointers and call the early_init function, * if such a function exists */ static int device_early_init(struct hl_device *hdev) { int i, rc; char workq_name[32]; switch (hdev->asic_type) { case ASIC_GOYA: goya_set_asic_funcs(hdev); strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name)); break; case ASIC_GAUDI: gaudi_set_asic_funcs(hdev); strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name)); break; case ASIC_GAUDI_SEC: gaudi_set_asic_funcs(hdev); strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name)); break; case ASIC_GAUDI2: gaudi2_set_asic_funcs(hdev); strscpy(hdev->asic_name, "GAUDI2", sizeof(hdev->asic_name)); break; case ASIC_GAUDI2B: gaudi2_set_asic_funcs(hdev); strscpy(hdev->asic_name, "GAUDI2B", sizeof(hdev->asic_name)); break; break; default: dev_err(hdev->dev, "Unrecognized ASIC type %d\n", hdev->asic_type); return -EINVAL; } rc = hdev->asic_funcs->early_init(hdev); if (rc) return rc; rc = hl_asid_init(hdev); if (rc) goto early_fini; if (hdev->asic_prop.completion_queues_count) { hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count, sizeof(struct workqueue_struct *), GFP_KERNEL); if (!hdev->cq_wq) { rc = -ENOMEM; goto asid_fini; } } for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i); hdev->cq_wq[i] = create_singlethread_workqueue(workq_name); if (hdev->cq_wq[i] == NULL) { dev_err(hdev->dev, "Failed to allocate CQ workqueue\n"); rc = -ENOMEM; goto free_cq_wq; } } hdev->eq_wq = create_singlethread_workqueue("hl-events"); if (hdev->eq_wq == NULL) { dev_err(hdev->dev, "Failed to allocate EQ workqueue\n"); rc = -ENOMEM; goto free_cq_wq; } hdev->cs_cmplt_wq = alloc_workqueue("hl-cs-completions", WQ_UNBOUND, 0); if (!hdev->cs_cmplt_wq) { dev_err(hdev->dev, "Failed to allocate CS completions workqueue\n"); rc = -ENOMEM; goto free_eq_wq; } hdev->ts_free_obj_wq = alloc_workqueue("hl-ts-free-obj", WQ_UNBOUND, 0); if (!hdev->ts_free_obj_wq) { dev_err(hdev->dev, "Failed to allocate Timestamp registration free workqueue\n"); rc = -ENOMEM; goto free_cs_cmplt_wq; } hdev->prefetch_wq = alloc_workqueue("hl-prefetch", WQ_UNBOUND, 0); if (!hdev->prefetch_wq) { dev_err(hdev->dev, "Failed to allocate MMU prefetch workqueue\n"); rc = -ENOMEM; goto free_ts_free_wq; } hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), GFP_KERNEL); if (!hdev->hl_chip_info) { rc = -ENOMEM; goto free_prefetch_wq; } rc = hl_mmu_if_set_funcs(hdev); if (rc) goto free_chip_info; hl_mem_mgr_init(hdev->dev, &hdev->kernel_mem_mgr); hdev->reset_wq = create_singlethread_workqueue("hl_device_reset"); if (!hdev->reset_wq) { rc = -ENOMEM; dev_err(hdev->dev, "Failed to create device reset WQ\n"); goto free_cb_mgr; } INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work, device_hard_reset_pending); hdev->device_reset_work.hdev = hdev; hdev->device_fini_pending = 0; INIT_DELAYED_WORK(&hdev->device_release_watchdog_work.reset_work, device_release_watchdog_func); hdev->device_release_watchdog_work.hdev = hdev; mutex_init(&hdev->send_cpu_message_lock); mutex_init(&hdev->debug_lock); INIT_LIST_HEAD(&hdev->cs_mirror_list); spin_lock_init(&hdev->cs_mirror_lock); spin_lock_init(&hdev->reset_info.lock); INIT_LIST_HEAD(&hdev->fpriv_list); INIT_LIST_HEAD(&hdev->fpriv_ctrl_list); mutex_init(&hdev->fpriv_list_lock); mutex_init(&hdev->fpriv_ctrl_list_lock); mutex_init(&hdev->clk_throttling.lock); return 0; free_cb_mgr: hl_mem_mgr_fini(&hdev->kernel_mem_mgr); free_chip_info: kfree(hdev->hl_chip_info); free_prefetch_wq: destroy_workqueue(hdev->prefetch_wq); free_ts_free_wq: destroy_workqueue(hdev->ts_free_obj_wq); free_cs_cmplt_wq: destroy_workqueue(hdev->cs_cmplt_wq); free_eq_wq: destroy_workqueue(hdev->eq_wq); free_cq_wq: for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) if (hdev->cq_wq[i]) destroy_workqueue(hdev->cq_wq[i]); kfree(hdev->cq_wq); asid_fini: hl_asid_fini(hdev); early_fini: if (hdev->asic_funcs->early_fini) hdev->asic_funcs->early_fini(hdev); return rc; } /* * device_early_fini - finalize all that was done in device_early_init * * @hdev: pointer to habanalabs device structure * */ static void device_early_fini(struct hl_device *hdev) { int i; mutex_destroy(&hdev->debug_lock); mutex_destroy(&hdev->send_cpu_message_lock); mutex_destroy(&hdev->fpriv_list_lock); mutex_destroy(&hdev->fpriv_ctrl_list_lock); mutex_destroy(&hdev->clk_throttling.lock); hl_mem_mgr_fini(&hdev->kernel_mem_mgr); kfree(hdev->hl_chip_info); destroy_workqueue(hdev->prefetch_wq); destroy_workqueue(hdev->ts_free_obj_wq); destroy_workqueue(hdev->cs_cmplt_wq); destroy_workqueue(hdev->eq_wq); destroy_workqueue(hdev->reset_wq); for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) destroy_workqueue(hdev->cq_wq[i]); kfree(hdev->cq_wq); hl_asid_fini(hdev); if (hdev->asic_funcs->early_fini) hdev->asic_funcs->early_fini(hdev); } static void hl_device_heartbeat(struct work_struct *work) { struct hl_device *hdev = container_of(work, struct hl_device, work_heartbeat.work); if (!hl_device_operational(hdev, NULL)) goto reschedule; if (!hdev->asic_funcs->send_heartbeat(hdev)) goto reschedule; if (hl_device_operational(hdev, NULL)) dev_err(hdev->dev, "Device heartbeat failed!\n"); hl_device_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT); return; reschedule: /* * prev_reset_trigger tracks consecutive fatal h/w errors until first * heartbeat immediately post reset. * If control reached here, then at least one heartbeat work has been * scheduled since last reset/init cycle. * So if the device is not already in reset cycle, reset the flag * prev_reset_trigger as no reset occurred with HL_DRV_RESET_FW_FATAL_ERR * status for at least one heartbeat. From this point driver restarts * tracking future consecutive fatal errors. */ if (!hdev->reset_info.in_reset) hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT; schedule_delayed_work(&hdev->work_heartbeat, usecs_to_jiffies(HL_HEARTBEAT_PER_USEC)); } /* * device_late_init - do late stuff initialization for the habanalabs device * * @hdev: pointer to habanalabs device structure * * Do stuff that either needs the device H/W queues to be active or needs * to happen after all the rest of the initialization is finished */ static int device_late_init(struct hl_device *hdev) { int rc; if (hdev->asic_funcs->late_init) { rc = hdev->asic_funcs->late_init(hdev); if (rc) { dev_err(hdev->dev, "failed late initialization for the H/W\n"); return rc; } } hdev->high_pll = hdev->asic_prop.high_pll; if (hdev->heartbeat) { INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat); schedule_delayed_work(&hdev->work_heartbeat, usecs_to_jiffies(HL_HEARTBEAT_PER_USEC)); } hdev->late_init_done = true; return 0; } /* * device_late_fini - finalize all that was done in device_late_init * * @hdev: pointer to habanalabs device structure * */ static void device_late_fini(struct hl_device *hdev) { if (!hdev->late_init_done) return; if (hdev->heartbeat) cancel_delayed_work_sync(&hdev->work_heartbeat); if (hdev->asic_funcs->late_fini) hdev->asic_funcs->late_fini(hdev); hdev->late_init_done = false; } int hl_device_utilization(struct hl_device *hdev, u32 *utilization) { u64 max_power, curr_power, dc_power, dividend, divisor; int rc; max_power = hdev->max_power; dc_power = hdev->asic_prop.dc_power_default; divisor = max_power - dc_power; if (!divisor) { dev_warn(hdev->dev, "device utilization is not supported\n"); return -EOPNOTSUPP; } rc = hl_fw_cpucp_power_get(hdev, &curr_power); if (rc) return rc; curr_power = clamp(curr_power, dc_power, max_power); dividend = (curr_power - dc_power) * 100; *utilization = (u32) div_u64(dividend, divisor); return 0; } int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable) { int rc = 0; mutex_lock(&hdev->debug_lock); if (!enable) { if (!hdev->in_debug) { dev_err(hdev->dev, "Failed to disable debug mode because device was not in debug mode\n"); rc = -EFAULT; goto out; } if (!hdev->reset_info.hard_reset_pending) hdev->asic_funcs->halt_coresight(hdev, ctx); hdev->in_debug = 0; goto out; } if (hdev->in_debug) { dev_err(hdev->dev, "Failed to enable debug mode because device is already in debug mode\n"); rc = -EFAULT; goto out; } hdev->in_debug = 1; out: mutex_unlock(&hdev->debug_lock); return rc; } static void take_release_locks(struct hl_device *hdev) { /* Flush anyone that is inside the critical section of enqueue * jobs to the H/W */ hdev->asic_funcs->hw_queues_lock(hdev); hdev->asic_funcs->hw_queues_unlock(hdev); /* Flush processes that are sending message to CPU */ mutex_lock(&hdev->send_cpu_message_lock); mutex_unlock(&hdev->send_cpu_message_lock); /* Flush anyone that is inside device open */ mutex_lock(&hdev->fpriv_list_lock); mutex_unlock(&hdev->fpriv_list_lock); mutex_lock(&hdev->fpriv_ctrl_list_lock); mutex_unlock(&hdev->fpriv_ctrl_list_lock); } static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset, bool skip_wq_flush) { if (hard_reset) device_late_fini(hdev); /* * Halt the engines and disable interrupts so we won't get any more * completions from H/W and we won't have any accesses from the * H/W to the host machine */ hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset); /* Go over all the queues, release all CS and their jobs */ hl_cs_rollback_all(hdev, skip_wq_flush); /* flush the MMU prefetch workqueue */ flush_workqueue(hdev->prefetch_wq); /* Release all pending user interrupts, each pending user interrupt * holds a reference to user context */ hl_release_pending_user_interrupts(hdev); } /* * hl_device_suspend - initiate device suspend * * @hdev: pointer to habanalabs device structure * * Puts the hw in the suspend state (all asics). * Returns 0 for success or an error on failure. * Called at driver suspend. */ int hl_device_suspend(struct hl_device *hdev) { int rc; pci_save_state(hdev->pdev); /* Block future CS/VM/JOB completion operations */ spin_lock(&hdev->reset_info.lock); if (hdev->reset_info.in_reset) { spin_unlock(&hdev->reset_info.lock); dev_err(hdev->dev, "Can't suspend while in reset\n"); return -EIO; } hdev->reset_info.in_reset = 1; spin_unlock(&hdev->reset_info.lock); /* This blocks all other stuff that is not blocked by in_reset */ hdev->disabled = true; take_release_locks(hdev); rc = hdev->asic_funcs->suspend(hdev); if (rc) dev_err(hdev->dev, "Failed to disable PCI access of device CPU\n"); /* Shut down the device */ pci_disable_device(hdev->pdev); pci_set_power_state(hdev->pdev, PCI_D3hot); return 0; } /* * hl_device_resume - initiate device resume * * @hdev: pointer to habanalabs device structure * * Bring the hw back to operating state (all asics). * Returns 0 for success or an error on failure. * Called at driver resume. */ int hl_device_resume(struct hl_device *hdev) { int rc; pci_set_power_state(hdev->pdev, PCI_D0); pci_restore_state(hdev->pdev); rc = pci_enable_device_mem(hdev->pdev); if (rc) { dev_err(hdev->dev, "Failed to enable PCI device in resume\n"); return rc; } pci_set_master(hdev->pdev); rc = hdev->asic_funcs->resume(hdev); if (rc) { dev_err(hdev->dev, "Failed to resume device after suspend\n"); goto disable_device; } /* 'in_reset' was set to true during suspend, now we must clear it in order * for hard reset to be performed */ spin_lock(&hdev->reset_info.lock); hdev->reset_info.in_reset = 0; spin_unlock(&hdev->reset_info.lock); rc = hl_device_reset(hdev, HL_DRV_RESET_HARD); if (rc) { dev_err(hdev->dev, "Failed to reset device during resume\n"); goto disable_device; } return 0; disable_device: pci_clear_master(hdev->pdev); pci_disable_device(hdev->pdev); return rc; } static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev) { struct task_struct *task = NULL; struct list_head *fd_list; struct hl_fpriv *hpriv; struct mutex *fd_lock; u32 pending_cnt; fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock; fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list; /* Giving time for user to close FD, and for processes that are inside * hl_device_open to finish */ if (!list_empty(fd_list)) ssleep(1); if (timeout) { pending_cnt = timeout; } else { if (hdev->process_kill_trial_cnt) { /* Processes have been already killed */ pending_cnt = 1; goto wait_for_processes; } else { /* Wait a small period after process kill */ pending_cnt = HL_PENDING_RESET_PER_SEC; } } mutex_lock(fd_lock); /* This section must be protected because we are dereferencing * pointers that are freed if the process exits */ list_for_each_entry(hpriv, fd_list, dev_node) { task = get_pid_task(hpriv->taskpid, PIDTYPE_PID); if (task) { dev_info(hdev->dev, "Killing user process pid=%d\n", task_pid_nr(task)); send_sig(SIGKILL, task, 1); usleep_range(1000, 10000); put_task_struct(task); } else { /* * If we got here, it means that process was killed from outside the driver * right after it started looping on fd_list and before get_pid_task, thus * we don't need to kill it. */ dev_dbg(hdev->dev, "Can't get task struct for user process, assuming process was killed from outside the driver\n"); } } mutex_unlock(fd_lock); /* * We killed the open users, but that doesn't mean they are closed. * It could be that they are running a long cleanup phase in the driver * e.g. MMU unmappings, or running other long teardown flow even before * our cleanup. * Therefore we need to wait again to make sure they are closed before * continuing with the reset. */ wait_for_processes: while ((!list_empty(fd_list)) && (pending_cnt)) { dev_dbg(hdev->dev, "Waiting for all unmap operations to finish before hard reset\n"); pending_cnt--; ssleep(1); } /* All processes exited successfully */ if (list_empty(fd_list)) return 0; /* Give up waiting for processes to exit */ if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS) return -ETIME; hdev->process_kill_trial_cnt++; return -EBUSY; } static void device_disable_open_processes(struct hl_device *hdev, bool control_dev) { struct list_head *fd_list; struct hl_fpriv *hpriv; struct mutex *fd_lock; fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock; fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list; mutex_lock(fd_lock); list_for_each_entry(hpriv, fd_list, dev_node) hpriv->hdev = NULL; mutex_unlock(fd_lock); } static void handle_reset_trigger(struct hl_device *hdev, u32 flags) { u32 cur_reset_trigger = HL_RESET_TRIGGER_DEFAULT; /* No consecutive mechanism when user context exists */ if (hdev->is_compute_ctx_active) return; /* * 'reset cause' is being updated here, because getting here * means that it's the 1st time and the last time we're here * ('in_reset' makes sure of it). This makes sure that * 'reset_cause' will continue holding its 1st recorded reason! */ if (flags & HL_DRV_RESET_HEARTBEAT) { hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT; cur_reset_trigger = HL_DRV_RESET_HEARTBEAT; } else if (flags & HL_DRV_RESET_TDR) { hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_TDR; cur_reset_trigger = HL_DRV_RESET_TDR; } else if (flags & HL_DRV_RESET_FW_FATAL_ERR) { hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; cur_reset_trigger = HL_DRV_RESET_FW_FATAL_ERR; } else { hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; } /* * If reset cause is same twice, then reset_trigger_repeated * is set and if this reset is due to a fatal FW error * device is set to an unstable state. */ if (hdev->reset_info.prev_reset_trigger != cur_reset_trigger) { hdev->reset_info.prev_reset_trigger = cur_reset_trigger; hdev->reset_info.reset_trigger_repeated = 0; } else { hdev->reset_info.reset_trigger_repeated = 1; } /* If reset is due to heartbeat, device CPU is no responsive in * which case no point sending PCI disable message to it. * * If F/W is performing the reset, no need to send it a message to disable * PCI access */ if ((flags & HL_DRV_RESET_HARD) && !(flags & (HL_DRV_RESET_HEARTBEAT | HL_DRV_RESET_BYPASS_REQ_TO_FW))) { /* Disable PCI access from device F/W so he won't send * us additional interrupts. We disable MSI/MSI-X at * the halt_engines function and we can't have the F/W * sending us interrupts after that. We need to disable * the access here because if the device is marked * disable, the message won't be send. Also, in case * of heartbeat, the device CPU is marked as disable * so this message won't be sent */ if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0)) dev_warn(hdev->dev, "Failed to disable PCI access by F/W\n"); } } /* * hl_device_reset - reset the device * * @hdev: pointer to habanalabs device structure * @flags: reset flags. * * Block future CS and wait for pending CS to be enqueued * Call ASIC H/W fini * Flush all completions * Re-initialize all internal data structures * Call ASIC H/W init, late_init * Test queues * Enable device * * Returns 0 for success or an error on failure. */ int hl_device_reset(struct hl_device *hdev, u32 flags) { bool hard_reset, from_hard_reset_thread, fw_reset, hard_instead_soft = false, reset_upon_device_release = false, schedule_hard_reset = false, delay_reset, from_dev_release, from_watchdog_thread; u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; struct hl_ctx *ctx; int i, rc; if (!hdev->init_done) { dev_err(hdev->dev, "Can't reset before initialization is done\n"); return 0; } hard_reset = !!(flags & HL_DRV_RESET_HARD); from_hard_reset_thread = !!(flags & HL_DRV_RESET_FROM_RESET_THR); fw_reset = !!(flags & HL_DRV_RESET_BYPASS_REQ_TO_FW); from_dev_release = !!(flags & HL_DRV_RESET_DEV_RELEASE); delay_reset = !!(flags & HL_DRV_RESET_DELAY); from_watchdog_thread = !!(flags & HL_DRV_RESET_FROM_WD_THR); if (!hard_reset && !hdev->asic_prop.supports_compute_reset) { hard_instead_soft = true; hard_reset = true; } if (hdev->reset_upon_device_release && (flags & HL_DRV_RESET_DEV_RELEASE)) { if (hard_reset) { dev_crit(hdev->dev, "Aborting reset because hard-reset is mutually exclusive with reset-on-device-release\n"); return -EINVAL; } reset_upon_device_release = true; goto do_reset; } if (!hard_reset && !hdev->asic_prop.allow_inference_soft_reset) { hard_instead_soft = true; hard_reset = true; } if (hard_instead_soft) dev_dbg(hdev->dev, "Doing hard-reset instead of compute reset\n"); do_reset: /* Re-entry of reset thread */ if (from_hard_reset_thread && hdev->process_kill_trial_cnt) goto kill_processes; /* * Prevent concurrency in this function - only one reset should be * done at any given time. Only need to perform this if we didn't * get from the dedicated hard reset thread */ if (!from_hard_reset_thread) { /* Block future CS/VM/JOB completion operations */ spin_lock(&hdev->reset_info.lock); if (hdev->reset_info.in_reset) { /* We only allow scheduling of a hard reset during compute reset */ if (hard_reset && hdev->reset_info.in_compute_reset) hdev->reset_info.hard_reset_schedule_flags = flags; spin_unlock(&hdev->reset_info.lock); return 0; } /* This still allows the completion of some KDMA ops * Update this before in_reset because in_compute_reset implies we are in reset */ hdev->reset_info.in_compute_reset = !hard_reset; hdev->reset_info.in_reset = 1; spin_unlock(&hdev->reset_info.lock); /* Cancel the device release watchdog work if required. * In case of reset-upon-device-release while the release watchdog work is * scheduled, do hard-reset instead of compute-reset. */ if ((hard_reset || from_dev_release) && hdev->reset_info.watchdog_active) { hdev->reset_info.watchdog_active = 0; if (!from_watchdog_thread) cancel_delayed_work_sync( &hdev->device_release_watchdog_work.reset_work); if (from_dev_release) { flags |= HL_DRV_RESET_HARD; flags &= ~HL_DRV_RESET_DEV_RELEASE; hard_reset = true; } } if (delay_reset) usleep_range(HL_RESET_DELAY_USEC, HL_RESET_DELAY_USEC << 1); handle_reset_trigger(hdev, flags); /* This also blocks future CS/VM/JOB completion operations */ hdev->disabled = true; take_release_locks(hdev); if (hard_reset) dev_info(hdev->dev, "Going to reset device\n"); else if (reset_upon_device_release) dev_dbg(hdev->dev, "Going to reset device after release by user\n"); else dev_dbg(hdev->dev, "Going to reset engines of inference device\n"); } again: if ((hard_reset) && (!from_hard_reset_thread)) { hdev->reset_info.hard_reset_pending = true; hdev->process_kill_trial_cnt = 0; hdev->device_reset_work.flags = flags; /* * Because the reset function can't run from heartbeat work, * we need to call the reset function from a dedicated work. */ queue_delayed_work(hdev->reset_wq, &hdev->device_reset_work.reset_work, 0); return 0; } cleanup_resources(hdev, hard_reset, fw_reset, from_dev_release); kill_processes: if (hard_reset) { /* Kill processes here after CS rollback. This is because the * process can't really exit until all its CSs are done, which * is what we do in cs rollback */ rc = device_kill_open_processes(hdev, 0, false); if (rc == -EBUSY) { if (hdev->device_fini_pending) { dev_crit(hdev->dev, "Failed to kill all open processes, stopping hard reset\n"); goto out_err; } /* signal reset thread to reschedule */ return rc; } if (rc) { dev_crit(hdev->dev, "Failed to kill all open processes, stopping hard reset\n"); goto out_err; } /* Flush the Event queue workers to make sure no other thread is * reading or writing to registers during the reset */ flush_workqueue(hdev->eq_wq); } /* Reset the H/W. It will be in idle state after this returns */ hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset); if (hard_reset) { hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE; /* Release kernel context */ if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1) hdev->kernel_ctx = NULL; hl_vm_fini(hdev); hl_mmu_fini(hdev); hl_eq_reset(hdev, &hdev->event_queue); } /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */ hl_hw_queue_reset(hdev, hard_reset); for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) hl_cq_reset(hdev, &hdev->completion_queue[i]); /* Make sure the context switch phase will run again */ ctx = hl_get_compute_ctx(hdev); if (ctx) { atomic_set(&ctx->thread_ctx_switch_token, 1); ctx->thread_ctx_switch_wait_token = 0; hl_ctx_put(ctx); } /* Finished tear-down, starting to re-initialize */ if (hard_reset) { hdev->device_cpu_disabled = false; hdev->reset_info.hard_reset_pending = false; if (hdev->reset_info.reset_trigger_repeated && (hdev->reset_info.prev_reset_trigger == HL_DRV_RESET_FW_FATAL_ERR)) { /* if there 2 back to back resets from FW, * ensure driver puts the driver in a unusable state */ dev_crit(hdev->dev, "Consecutive FW fatal errors received, stopping hard reset\n"); rc = -EIO; goto out_err; } if (hdev->kernel_ctx) { dev_crit(hdev->dev, "kernel ctx was alive during hard reset, something is terribly wrong\n"); rc = -EBUSY; goto out_err; } rc = hl_mmu_init(hdev); if (rc) { dev_err(hdev->dev, "Failed to initialize MMU S/W after hard reset\n"); goto out_err; } /* Allocate the kernel context */ hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL); if (!hdev->kernel_ctx) { rc = -ENOMEM; hl_mmu_fini(hdev); goto out_err; } hdev->is_compute_ctx_active = false; rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); if (rc) { dev_err(hdev->dev, "failed to init kernel ctx in hard reset\n"); kfree(hdev->kernel_ctx); hdev->kernel_ctx = NULL; hl_mmu_fini(hdev); goto out_err; } } /* Device is now enabled as part of the initialization requires * communication with the device firmware to get information that * is required for the initialization itself */ hdev->disabled = false; /* F/W security enabled indication might be updated after hard-reset */ if (hard_reset) { rc = hl_fw_read_preboot_status(hdev); if (rc) goto out_err; } rc = hdev->asic_funcs->hw_init(hdev); if (rc) { dev_err(hdev->dev, "failed to initialize the H/W after reset\n"); goto out_err; } /* If device is not idle fail the reset process */ if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask, HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) { print_idle_status_mask(hdev, "device is not idle after reset", idle_mask); rc = -EIO; goto out_err; } /* Check that the communication with the device is working */ rc = hdev->asic_funcs->test_queues(hdev); if (rc) { dev_err(hdev->dev, "Failed to detect if device is alive after reset\n"); goto out_err; } if (hard_reset) { rc = device_late_init(hdev); if (rc) { dev_err(hdev->dev, "Failed late init after hard reset\n"); goto out_err; } rc = hl_vm_init(hdev); if (rc) { dev_err(hdev->dev, "Failed to init memory module after hard reset\n"); goto out_err; } if (!hdev->asic_prop.fw_security_enabled) hl_fw_set_max_power(hdev); } else { rc = hdev->asic_funcs->compute_reset_late_init(hdev); if (rc) { if (reset_upon_device_release) dev_err(hdev->dev, "Failed late init in reset after device release\n"); else dev_err(hdev->dev, "Failed late init after compute reset\n"); goto out_err; } } rc = hdev->asic_funcs->scrub_device_mem(hdev); if (rc) { dev_err(hdev->dev, "scrub mem failed from device reset (%d)\n", rc); return rc; } spin_lock(&hdev->reset_info.lock); hdev->reset_info.in_compute_reset = 0; /* Schedule hard reset only if requested and if not already in hard reset. * We keep 'in_reset' enabled, so no other reset can go in during the hard * reset schedule */ if (!hard_reset && hdev->reset_info.hard_reset_schedule_flags) schedule_hard_reset = true; else hdev->reset_info.in_reset = 0; spin_unlock(&hdev->reset_info.lock); hdev->reset_info.needs_reset = false; if (hard_reset) dev_info(hdev->dev, "Successfully finished resetting the device\n"); else dev_dbg(hdev->dev, "Successfully finished resetting the device\n"); if (hard_reset) { hdev->reset_info.hard_reset_cnt++; /* After reset is done, we are ready to receive events from * the F/W. We can't do it before because we will ignore events * and if those events are fatal, we won't know about it and * the device will be operational although it shouldn't be */ hdev->asic_funcs->enable_events_from_fw(hdev); } else { if (!reset_upon_device_release) hdev->reset_info.compute_reset_cnt++; if (schedule_hard_reset) { dev_info(hdev->dev, "Performing hard reset scheduled during compute reset\n"); flags = hdev->reset_info.hard_reset_schedule_flags; hdev->reset_info.hard_reset_schedule_flags = 0; hdev->disabled = true; hard_reset = true; handle_reset_trigger(hdev, flags); goto again; } } return 0; out_err: hdev->disabled = true; spin_lock(&hdev->reset_info.lock); hdev->reset_info.in_compute_reset = 0; if (hard_reset) { dev_err(hdev->dev, "Failed to reset! Device is NOT usable\n"); hdev->reset_info.hard_reset_cnt++; } else if (reset_upon_device_release) { spin_unlock(&hdev->reset_info.lock); dev_err(hdev->dev, "Failed to reset device after user release\n"); flags |= HL_DRV_RESET_HARD; flags &= ~HL_DRV_RESET_DEV_RELEASE; hard_reset = true; goto again; } else { spin_unlock(&hdev->reset_info.lock); dev_err(hdev->dev, "Failed to do compute reset\n"); hdev->reset_info.compute_reset_cnt++; flags |= HL_DRV_RESET_HARD; hard_reset = true; goto again; } hdev->reset_info.in_reset = 0; spin_unlock(&hdev->reset_info.lock); return rc; } /* * hl_device_cond_reset() - conditionally reset the device. * @hdev: pointer to habanalabs device structure. * @reset_flags: reset flags. * @event_mask: events to notify user about. * * Conditionally reset the device, or alternatively schedule a watchdog work to reset the device * unless another reset precedes it. */ int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask) { struct hl_ctx *ctx = NULL; /* Device release watchdog is only for hard reset */ if (!(flags & HL_DRV_RESET_HARD) && hdev->asic_prop.allow_inference_soft_reset) goto device_reset; /* F/W reset cannot be postponed */ if (flags & HL_DRV_RESET_BYPASS_REQ_TO_FW) goto device_reset; /* Device release watchdog is relevant only if user exists and gets a reset notification */ if (!(event_mask & HL_NOTIFIER_EVENT_DEVICE_RESET)) { dev_err(hdev->dev, "Resetting device without a reset indication to user\n"); goto device_reset; } ctx = hl_get_compute_ctx(hdev); if (!ctx || !ctx->hpriv->notifier_event.eventfd) goto device_reset; /* Schedule the device release watchdog work unless reset is already in progress or if the * work is already scheduled. */ spin_lock(&hdev->reset_info.lock); if (hdev->reset_info.in_reset) { spin_unlock(&hdev->reset_info.lock); goto device_reset; } if (hdev->reset_info.watchdog_active) goto out; hdev->device_release_watchdog_work.flags = flags; dev_dbg(hdev->dev, "Device is going to be reset in %u sec unless being released\n", hdev->device_release_watchdog_timeout_sec); schedule_delayed_work(&hdev->device_release_watchdog_work.reset_work, msecs_to_jiffies(hdev->device_release_watchdog_timeout_sec * 1000)); hdev->reset_info.watchdog_active = 1; out: spin_unlock(&hdev->reset_info.lock); hl_notifier_event_send_all(hdev, event_mask); hl_ctx_put(ctx); return 0; device_reset: if (event_mask) hl_notifier_event_send_all(hdev, event_mask); if (ctx) hl_ctx_put(ctx); return hl_device_reset(hdev, flags); } static void hl_notifier_event_send(struct hl_notifier_event *notifier_event, u64 event_mask) { mutex_lock(¬ifier_event->lock); notifier_event->events_mask |= event_mask; if (notifier_event->eventfd) eventfd_signal(notifier_event->eventfd, 1); mutex_unlock(¬ifier_event->lock); } /* * hl_notifier_event_send_all - notify all user processes via eventfd * * @hdev: pointer to habanalabs device structure * @event_mask: the occurred event/s * Returns 0 for success or an error on failure. */ void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask) { struct hl_fpriv *hpriv; if (!event_mask) { dev_warn(hdev->dev, "Skip sending zero event"); return; } mutex_lock(&hdev->fpriv_list_lock); list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) hl_notifier_event_send(&hpriv->notifier_event, event_mask); mutex_unlock(&hdev->fpriv_list_lock); /* control device */ mutex_lock(&hdev->fpriv_ctrl_list_lock); list_for_each_entry(hpriv, &hdev->fpriv_ctrl_list, dev_node) hl_notifier_event_send(&hpriv->notifier_event, event_mask); mutex_unlock(&hdev->fpriv_ctrl_list_lock); } /* * hl_device_init - main initialization function for habanalabs device * * @hdev: pointer to habanalabs device structure * * Allocate an id for the device, do early initialization and then call the * ASIC specific initialization functions. Finally, create the cdev and the * Linux device to expose it to the user */ int hl_device_init(struct hl_device *hdev, struct class *hclass) { int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt; char *name; bool add_cdev_sysfs_on_err = false; hdev->cdev_idx = hdev->id / 2; name = kasprintf(GFP_KERNEL, "hl%d", hdev->cdev_idx); if (!name) { rc = -ENOMEM; goto out_disabled; } /* Initialize cdev and device structures */ rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name, &hdev->cdev, &hdev->dev); kfree(name); if (rc) goto out_disabled; name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->cdev_idx); if (!name) { rc = -ENOMEM; goto free_dev; } /* Initialize cdev and device structures for control device */ rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops, name, &hdev->cdev_ctrl, &hdev->dev_ctrl); kfree(name); if (rc) goto free_dev; /* Initialize ASIC function pointers and perform early init */ rc = device_early_init(hdev); if (rc) goto free_dev_ctrl; user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count + hdev->asic_prop.user_interrupt_count; if (user_interrupt_cnt) { hdev->user_interrupt = kcalloc(user_interrupt_cnt, sizeof(*hdev->user_interrupt), GFP_KERNEL); if (!hdev->user_interrupt) { rc = -ENOMEM; goto early_fini; } } /* * Start calling ASIC initialization. First S/W then H/W and finally * late init */ rc = hdev->asic_funcs->sw_init(hdev); if (rc) goto free_usr_intr_mem; /* initialize completion structure for multi CS wait */ hl_multi_cs_completion_init(hdev); /* * Initialize the H/W queues. Must be done before hw_init, because * there the addresses of the kernel queue are being written to the * registers of the device */ rc = hl_hw_queues_create(hdev); if (rc) { dev_err(hdev->dev, "failed to initialize kernel queues\n"); goto sw_fini; } cq_cnt = hdev->asic_prop.completion_queues_count; /* * Initialize the completion queues. Must be done before hw_init, * because there the addresses of the completion queues are being * passed as arguments to request_irq */ if (cq_cnt) { hdev->completion_queue = kcalloc(cq_cnt, sizeof(*hdev->completion_queue), GFP_KERNEL); if (!hdev->completion_queue) { dev_err(hdev->dev, "failed to allocate completion queues\n"); rc = -ENOMEM; goto hw_queues_destroy; } } for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) { rc = hl_cq_init(hdev, &hdev->completion_queue[i], hdev->asic_funcs->get_queue_id_for_cq(hdev, i)); if (rc) { dev_err(hdev->dev, "failed to initialize completion queue\n"); goto cq_fini; } hdev->completion_queue[i].cq_idx = i; } hdev->shadow_cs_queue = kcalloc(hdev->asic_prop.max_pending_cs, sizeof(struct hl_cs *), GFP_KERNEL); if (!hdev->shadow_cs_queue) { rc = -ENOMEM; goto cq_fini; } /* * Initialize the event queue. Must be done before hw_init, * because there the address of the event queue is being * passed as argument to request_irq */ rc = hl_eq_init(hdev, &hdev->event_queue); if (rc) { dev_err(hdev->dev, "failed to initialize event queue\n"); goto free_shadow_cs_queue; } /* MMU S/W must be initialized before kernel context is created */ rc = hl_mmu_init(hdev); if (rc) { dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n"); goto eq_fini; } /* Allocate the kernel context */ hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL); if (!hdev->kernel_ctx) { rc = -ENOMEM; goto mmu_fini; } hdev->is_compute_ctx_active = false; hdev->asic_funcs->state_dump_init(hdev); hdev->device_release_watchdog_timeout_sec = HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC; hdev->memory_scrub_val = MEM_SCRUB_DEFAULT_VAL; hl_debugfs_add_device(hdev); /* debugfs nodes are created in hl_ctx_init so it must be called after * hl_debugfs_add_device. */ rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); if (rc) { dev_err(hdev->dev, "failed to initialize kernel context\n"); kfree(hdev->kernel_ctx); goto remove_device_from_debugfs; } rc = hl_cb_pool_init(hdev); if (rc) { dev_err(hdev->dev, "failed to initialize CB pool\n"); goto release_ctx; } rc = hl_dec_init(hdev); if (rc) { dev_err(hdev->dev, "Failed to initialize the decoder module\n"); goto cb_pool_fini; } /* * From this point, override rc (=0) in case of an error to allow * debugging (by adding char devices and create sysfs nodes as part of * the error flow). */ add_cdev_sysfs_on_err = true; /* Device is now enabled as part of the initialization requires * communication with the device firmware to get information that * is required for the initialization itself */ hdev->disabled = false; rc = hdev->asic_funcs->hw_init(hdev); if (rc) { dev_err(hdev->dev, "failed to initialize the H/W\n"); rc = 0; goto out_disabled; } /* Check that the communication with the device is working */ rc = hdev->asic_funcs->test_queues(hdev); if (rc) { dev_err(hdev->dev, "Failed to detect if device is alive\n"); rc = 0; goto out_disabled; } rc = device_late_init(hdev); if (rc) { dev_err(hdev->dev, "Failed late initialization\n"); rc = 0; goto out_disabled; } dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n", hdev->asic_name, hdev->asic_prop.dram_size / SZ_1G); rc = hl_vm_init(hdev); if (rc) { dev_err(hdev->dev, "Failed to initialize memory module\n"); rc = 0; goto out_disabled; } /* * Expose devices and sysfs nodes to user. * From here there is no need to add char devices and create sysfs nodes * in case of an error. */ add_cdev_sysfs_on_err = false; rc = device_cdev_sysfs_add(hdev); if (rc) { dev_err(hdev->dev, "Failed to add char devices and sysfs nodes\n"); rc = 0; goto out_disabled; } /* Need to call this again because the max power might change, * depending on card type for certain ASICs */ if (hdev->asic_prop.set_max_power_on_device_init && !hdev->asic_prop.fw_security_enabled) hl_fw_set_max_power(hdev); /* * hl_hwmon_init() must be called after device_late_init(), because only * there we get the information from the device about which * hwmon-related sensors the device supports. * Furthermore, it must be done after adding the device to the system. */ rc = hl_hwmon_init(hdev); if (rc) { dev_err(hdev->dev, "Failed to initialize hwmon\n"); rc = 0; goto out_disabled; } dev_notice(hdev->dev, "Successfully added device to habanalabs driver\n"); hdev->init_done = true; /* After initialization is done, we are ready to receive events from * the F/W. We can't do it before because we will ignore events and if * those events are fatal, we won't know about it and the device will * be operational although it shouldn't be */ hdev->asic_funcs->enable_events_from_fw(hdev); return 0; cb_pool_fini: hl_cb_pool_fini(hdev); release_ctx: if (hl_ctx_put(hdev->kernel_ctx) != 1) dev_err(hdev->dev, "kernel ctx is still alive on initialization failure\n"); remove_device_from_debugfs: hl_debugfs_remove_device(hdev); mmu_fini: hl_mmu_fini(hdev); eq_fini: hl_eq_fini(hdev, &hdev->event_queue); free_shadow_cs_queue: kfree(hdev->shadow_cs_queue); cq_fini: for (i = 0 ; i < cq_ready_cnt ; i++) hl_cq_fini(hdev, &hdev->completion_queue[i]); kfree(hdev->completion_queue); hw_queues_destroy: hl_hw_queues_destroy(hdev); sw_fini: hdev->asic_funcs->sw_fini(hdev); free_usr_intr_mem: kfree(hdev->user_interrupt); early_fini: device_early_fini(hdev); free_dev_ctrl: put_device(hdev->dev_ctrl); free_dev: put_device(hdev->dev); out_disabled: hdev->disabled = true; if (add_cdev_sysfs_on_err) device_cdev_sysfs_add(hdev); if (hdev->pdev) dev_err(&hdev->pdev->dev, "Failed to initialize hl%d. Device is NOT usable !\n", hdev->cdev_idx); else pr_err("Failed to initialize hl%d. Device is NOT usable !\n", hdev->cdev_idx); return rc; } /* * hl_device_fini - main tear-down function for habanalabs device * * @hdev: pointer to habanalabs device structure * * Destroy the device, call ASIC fini functions and release the id */ void hl_device_fini(struct hl_device *hdev) { bool device_in_reset; ktime_t timeout; u64 reset_sec; int i, rc; dev_info(hdev->dev, "Removing device\n"); hdev->device_fini_pending = 1; flush_delayed_work(&hdev->device_reset_work.reset_work); if (hdev->pldm) reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT; else reset_sec = HL_HARD_RESET_MAX_TIMEOUT; /* * This function is competing with the reset function, so try to * take the reset atomic and if we are already in middle of reset, * wait until reset function is finished. Reset function is designed * to always finish. However, in Gaudi, because of all the network * ports, the hard reset could take between 10-30 seconds */ timeout = ktime_add_us(ktime_get(), reset_sec * 1000 * 1000); spin_lock(&hdev->reset_info.lock); device_in_reset = !!hdev->reset_info.in_reset; if (!device_in_reset) hdev->reset_info.in_reset = 1; spin_unlock(&hdev->reset_info.lock); while (device_in_reset) { usleep_range(50, 200); spin_lock(&hdev->reset_info.lock); device_in_reset = !!hdev->reset_info.in_reset; if (!device_in_reset) hdev->reset_info.in_reset = 1; spin_unlock(&hdev->reset_info.lock); if (ktime_compare(ktime_get(), timeout) > 0) { dev_crit(hdev->dev, "Failed to remove device because reset function did not finish\n"); return; } } cancel_delayed_work_sync(&hdev->device_release_watchdog_work.reset_work); /* Disable PCI access from device F/W so it won't send us additional * interrupts. We disable MSI/MSI-X at the halt_engines function and we * can't have the F/W sending us interrupts after that. We need to * disable the access here because if the device is marked disable, the * message won't be send. Also, in case of heartbeat, the device CPU is * marked as disable so this message won't be sent */ hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); /* Mark device as disabled */ hdev->disabled = true; take_release_locks(hdev); hdev->reset_info.hard_reset_pending = true; hl_hwmon_fini(hdev); cleanup_resources(hdev, true, false, false); /* Kill processes here after CS rollback. This is because the process * can't really exit until all its CSs are done, which is what we * do in cs rollback */ dev_info(hdev->dev, "Waiting for all processes to exit (timeout of %u seconds)", HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI); hdev->process_kill_trial_cnt = 0; rc = device_kill_open_processes(hdev, HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI, false); if (rc) { dev_crit(hdev->dev, "Failed to kill all open processes\n"); device_disable_open_processes(hdev, false); } hdev->process_kill_trial_cnt = 0; rc = device_kill_open_processes(hdev, 0, true); if (rc) { dev_crit(hdev->dev, "Failed to kill all control device open processes\n"); device_disable_open_processes(hdev, true); } hl_cb_pool_fini(hdev); /* Reset the H/W. It will be in idle state after this returns */ hdev->asic_funcs->hw_fini(hdev, true, false); hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE; /* Release kernel context */ if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1)) dev_err(hdev->dev, "kernel ctx is still alive\n"); hl_debugfs_remove_device(hdev); hl_dec_fini(hdev); hl_vm_fini(hdev); hl_mmu_fini(hdev); vfree(hdev->captured_err_info.pgf_info.user_mappings); hl_eq_fini(hdev, &hdev->event_queue); kfree(hdev->shadow_cs_queue); for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) hl_cq_fini(hdev, &hdev->completion_queue[i]); kfree(hdev->completion_queue); kfree(hdev->user_interrupt); hl_hw_queues_destroy(hdev); /* Call ASIC S/W finalize function */ hdev->asic_funcs->sw_fini(hdev); device_early_fini(hdev); /* Hide devices and sysfs nodes from user */ device_cdev_sysfs_del(hdev); pr_info("removed device successfully\n"); } /* * MMIO register access helper functions. */ /* * hl_rreg - Read an MMIO register * * @hdev: pointer to habanalabs device structure * @reg: MMIO register offset (in bytes) * * Returns the value of the MMIO register we are asked to read * */ inline u32 hl_rreg(struct hl_device *hdev, u32 reg) { return readl(hdev->rmmio + reg); } /* * hl_wreg - Write to an MMIO register * * @hdev: pointer to habanalabs device structure * @reg: MMIO register offset (in bytes) * @val: 32-bit value * * Writes the 32-bit value into the MMIO register * */ inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val) { writel(val, hdev->rmmio + reg); } void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, u8 flags) { if (num_of_engines > HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR) { dev_err(hdev->dev, "Number of possible razwi initiators (%u) exceeded limit (%u)\n", num_of_engines, HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR); return; } /* In case it's the first razwi since the device was opened, capture its parameters */ if (atomic_cmpxchg(&hdev->captured_err_info.razwi_info_recorded, 0, 1)) return; hdev->captured_err_info.razwi.timestamp = ktime_to_ns(ktime_get()); hdev->captured_err_info.razwi.addr = addr; hdev->captured_err_info.razwi.num_of_possible_engines = num_of_engines; memcpy(&hdev->captured_err_info.razwi.engine_id[0], &engine_id[0], num_of_engines * sizeof(u16)); hdev->captured_err_info.razwi.flags = flags; } void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, u8 flags, u64 *event_mask) { hl_capture_razwi(hdev, addr, engine_id, num_of_engines, flags); if (event_mask) *event_mask |= HL_NOTIFIER_EVENT_RAZWI; } static void hl_capture_user_mappings(struct hl_device *hdev, bool is_pmmu) { struct page_fault_info *pgf_info = &hdev->captured_err_info.pgf_info; struct hl_vm_phys_pg_pack *phys_pg_pack = NULL; struct hl_vm_hash_node *hnode; struct hl_userptr *userptr; enum vm_type *vm_type; struct hl_ctx *ctx; u32 map_idx = 0; int i; /* Reset previous session count*/ pgf_info->num_of_user_mappings = 0; ctx = hl_get_compute_ctx(hdev); if (!ctx) { dev_err(hdev->dev, "Can't get user context for user mappings\n"); return; } mutex_lock(&ctx->mem_hash_lock); hash_for_each(ctx->mem_hash, i, hnode, node) { vm_type = hnode->ptr; if (((*vm_type == VM_TYPE_USERPTR) && is_pmmu) || ((*vm_type == VM_TYPE_PHYS_PACK) && !is_pmmu)) pgf_info->num_of_user_mappings++; } if (!pgf_info->num_of_user_mappings) goto finish; /* In case we already allocated in previous session, need to release it before * allocating new buffer. */ vfree(pgf_info->user_mappings); pgf_info->user_mappings = vzalloc(pgf_info->num_of_user_mappings * sizeof(struct hl_user_mapping)); if (!pgf_info->user_mappings) { pgf_info->num_of_user_mappings = 0; goto finish; } hash_for_each(ctx->mem_hash, i, hnode, node) { vm_type = hnode->ptr; if ((*vm_type == VM_TYPE_USERPTR) && (is_pmmu)) { userptr = hnode->ptr; pgf_info->user_mappings[map_idx].dev_va = hnode->vaddr; pgf_info->user_mappings[map_idx].size = userptr->size; map_idx++; } else if ((*vm_type == VM_TYPE_PHYS_PACK) && (!is_pmmu)) { phys_pg_pack = hnode->ptr; pgf_info->user_mappings[map_idx].dev_va = hnode->vaddr; pgf_info->user_mappings[map_idx].size = phys_pg_pack->total_size; map_idx++; } } finish: mutex_unlock(&ctx->mem_hash_lock); hl_ctx_put(ctx); } void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu) { /* Capture only the first page fault */ if (atomic_cmpxchg(&hdev->captured_err_info.pgf_info_recorded, 0, 1)) return; hdev->captured_err_info.pgf_info.pgf.timestamp = ktime_to_ns(ktime_get()); hdev->captured_err_info.pgf_info.pgf.addr = addr; hdev->captured_err_info.pgf_info.pgf.engine_id = eng_id; hl_capture_user_mappings(hdev, is_pmmu); } void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu, u64 *event_mask) { hl_capture_page_fault(hdev, addr, eng_id, is_pmmu); if (event_mask) *event_mask |= HL_NOTIFIER_EVENT_PAGE_FAULT; }