From 089f16c55baacd5e8ae3745625efa82899b4b217 Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 6 Jun 2016 10:17:50 +0200 Subject: drm/ttm: cleanup ttm_tt_(unbind|destroy) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ttm_tt_destroy should be the only one unbinding the object. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 2 -- drivers/gpu/drm/ttm/ttm_bo_util.c | 3 --- drivers/gpu/drm/ttm/ttm_tt.c | 17 +++++------------ 3 files changed, 5 insertions(+), 17 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 39386f50af87..4216b3162a8d 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -397,7 +397,6 @@ moved: out_err: new_man = &bdev->man[bo->mem.mem_type]; if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { - ttm_tt_unbind(bo->ttm); ttm_tt_destroy(bo->ttm); bo->ttm = NULL; } @@ -419,7 +418,6 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) bo->bdev->driver->move_notify(bo, NULL); if (bo->ttm) { - ttm_tt_unbind(bo->ttm); ttm_tt_destroy(bo->ttm); bo->ttm = NULL; } diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index d9831559706e..4194b7ea7a72 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -53,7 +53,6 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, int ret; if (old_mem->mem_type != TTM_PL_SYSTEM) { - ttm_tt_unbind(ttm); ttm_bo_free_old_node(bo); ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, TTM_PL_MASK_MEM); @@ -402,7 +401,6 @@ out2: new_mem->mm_node = NULL; if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) { - ttm_tt_unbind(ttm); ttm_tt_destroy(ttm); bo->ttm = NULL; } @@ -651,7 +649,6 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (bo->ttm != NULL)) { - ttm_tt_unbind(bo->ttm); ttm_tt_destroy(bo->ttm); bo->ttm = NULL; } diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 077ae9b2865d..79f6323771e2 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -166,11 +166,15 @@ EXPORT_SYMBOL(ttm_tt_set_placement_caching); void ttm_tt_destroy(struct ttm_tt *ttm) { + int ret; + if (unlikely(ttm == NULL)) return; if (ttm->state == tt_bound) { - ttm_tt_unbind(ttm); + ret = ttm->func->unbind(ttm); + BUG_ON(ret); + ttm->state = tt_unbound; } if (ttm->state == tt_unbound) @@ -251,17 +255,6 @@ void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) } EXPORT_SYMBOL(ttm_dma_tt_fini); -void ttm_tt_unbind(struct ttm_tt *ttm) -{ - int ret; - - if (ttm->state == tt_bound) { - ret = ttm->func->unbind(ttm); - BUG_ON(ret); - ttm->state = tt_unbound; - } -} - int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) { int ret = 0; -- cgit v1.2.3 From 4279cb1423d96e53b6b98ae9f2b41003b013a31f Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 6 Jun 2016 10:17:51 +0200 Subject: drm/ttm: remove NULL checks when calling ttm_tt_destroy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function is a no-op with a NULL pointer. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 11 ++++------- drivers/gpu/drm/ttm/ttm_bo_util.c | 5 ++--- drivers/gpu/drm/ttm/ttm_tt.c | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 4216b3162a8d..43a29555032e 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -147,8 +147,7 @@ static void ttm_bo_release_list(struct kref *list_kref) BUG_ON(!list_empty(&bo->lru)); BUG_ON(!list_empty(&bo->ddestroy)); - if (bo->ttm) - ttm_tt_destroy(bo->ttm); + ttm_tt_destroy(bo->ttm); atomic_dec(&bo->glob->bo_count); if (bo->resv == &bo->ttm_resv) reservation_object_fini(&bo->ttm_resv); @@ -396,7 +395,7 @@ moved: out_err: new_man = &bdev->man[bo->mem.mem_type]; - if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { + if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) { ttm_tt_destroy(bo->ttm); bo->ttm = NULL; } @@ -417,10 +416,8 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) if (bo->bdev->driver->move_notify) bo->bdev->driver->move_notify(bo, NULL); - if (bo->ttm) { - ttm_tt_destroy(bo->ttm); - bo->ttm = NULL; - } + ttm_tt_destroy(bo->ttm); + bo->ttm = NULL; ttm_bo_mem_put(bo, &bo->mem); ww_mutex_unlock (&bo->resv->lock); diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 4194b7ea7a72..0f4bcb0d594b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -400,7 +400,7 @@ out2: *old_mem = *new_mem; new_mem->mm_node = NULL; - if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) { + if (man->flags & TTM_MEMTYPE_FLAG_FIXED) { ttm_tt_destroy(ttm); bo->ttm = NULL; } @@ -647,8 +647,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, if (ret) return ret; - if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && - (bo->ttm != NULL)) { + if (man->flags & TTM_MEMTYPE_FLAG_FIXED) { ttm_tt_destroy(bo->ttm); bo->ttm = NULL; } diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 79f6323771e2..d28d4333dcce 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -168,7 +168,7 @@ void ttm_tt_destroy(struct ttm_tt *ttm) { int ret; - if (unlikely(ttm == NULL)) + if (ttm == NULL) return; if (ttm->state == tt_bound) { -- cgit v1.2.3 From 77dfc28bad2c75493125ba8660e4c27c2dcdab57 Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 6 Jun 2016 10:17:54 +0200 Subject: drm/ttm: wait for BO idle in ttm_bo_move_memcpy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When we want to pipeline accelerated moves we need to wait in the fallback path. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 3 ++- drivers/gpu/drm/nouveau/nouveau_bo.c | 2 +- drivers/gpu/drm/qxl/qxl_ttm.c | 3 ++- drivers/gpu/drm/radeon/radeon_ttm.c | 3 ++- drivers/gpu/drm/ttm/ttm_bo.c | 3 ++- drivers/gpu/drm/ttm/ttm_bo_util.c | 7 ++++++- include/drm/ttm/ttm_bo_driver.h | 4 +++- 7 files changed, 18 insertions(+), 7 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 0a6a6320b40e..9b244c5a6295 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -433,7 +433,8 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, if (r) { memcpy: - r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); + r = ttm_bo_move_memcpy(bo, evict, interruptible, + no_wait_gpu, new_mem); if (r) { return r; } diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index a43f3095693b..4fb1bf9b81b3 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -1328,7 +1328,7 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr, /* Fallback to software copy. */ ret = ttm_bo_wait(bo, intr, no_wait_gpu); if (ret == 0) - ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); + ret = ttm_bo_move_memcpy(bo, evict, intr, no_wait_gpu, new_mem); out: if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) { diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c index 89e8be944330..d50c9679e631 100644 --- a/drivers/gpu/drm/qxl/qxl_ttm.c +++ b/drivers/gpu/drm/qxl/qxl_ttm.c @@ -361,7 +361,8 @@ static int qxl_bo_move(struct ttm_buffer_object *bo, qxl_move_null(bo, new_mem); return 0; } - return ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); + return ttm_bo_move_memcpy(bo, evict, interruptible, + no_wait_gpu, new_mem); } static void qxl_bo_move_notify(struct ttm_buffer_object *bo, diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index 1cc4870b3de8..20ca22dc6813 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -445,7 +445,8 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, if (r) { memcpy: - r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); + r = ttm_bo_move_memcpy(bo, evict, interruptible, + no_wait_gpu, new_mem); if (r) { return r; } diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 43a29555032e..041fb3b8b2aa 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -359,7 +359,8 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, ret = bdev->driver->move(bo, evict, interruptible, no_wait_gpu, mem); else - ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); + ret = ttm_bo_move_memcpy(bo, evict, interruptible, + no_wait_gpu, mem); if (ret) { if (bdev->driver->move_notify) { diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 0f4bcb0d594b..434f2394ad2c 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -320,7 +320,8 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, } int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, - bool evict, bool no_wait_gpu, + bool evict, bool interruptible, + bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct ttm_bo_device *bdev = bo->bdev; @@ -336,6 +337,10 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, unsigned long add = 0; int dir; + ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); + if (ret) + return ret; + ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); if (ret) return ret; diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index da6ee17d7c0b..0d1d9d78c650 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -970,6 +970,7 @@ extern int ttm_bo_move_ttm(struct ttm_buffer_object *bo, * * @bo: A pointer to a struct ttm_buffer_object. * @evict: 1: This is an eviction. Don't try to pipeline. + * @interruptible: Sleep interruptible if waiting. * @no_wait_gpu: Return immediately if the GPU is busy. * @new_mem: struct ttm_mem_reg indicating where to move. * @@ -984,7 +985,8 @@ extern int ttm_bo_move_ttm(struct ttm_buffer_object *bo, */ extern int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, - bool evict, bool no_wait_gpu, + bool evict, bool interruptible, + bool no_wait_gpu, struct ttm_mem_reg *new_mem); /** -- cgit v1.2.3 From a6f76dcd02f36faa9d45b759732453dcfbf8fe8a Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 6 Jun 2016 10:17:55 +0200 Subject: drm/ttm: drop wait for idle in ttm_bo_move_buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit That is unnecessary now. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 041fb3b8b2aa..cc94df00939a 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -996,20 +996,6 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, lockdep_assert_held(&bo->resv->lock.base); - /* - * Don't wait for the BO on initial allocation. This is important when - * the BO has an imported reservation object. - */ - if (bo->mem.mem_type != TTM_PL_SYSTEM || bo->ttm != NULL) { - /* - * FIXME: It's possible to pipeline buffer moves. - * Have the driver move function wait for idle when necessary, - * instead of doing it here. - */ - ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); - if (ret) - return ret; - } mem.num_pages = bo->num_pages; mem.size = mem.num_pages << PAGE_SHIFT; mem.page_alignment = bo->mem.page_alignment; -- cgit v1.2.3 From 17d33bc9d6ef96008c8e6ee062da8d955fae97f7 Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 6 Jun 2016 10:17:56 +0200 Subject: drm/ttm: drop waiting for idle in ttm_bo_evict. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit That is unnecessary now. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index cc94df00939a..86dd695021a6 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -684,15 +684,6 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, struct ttm_placement placement; int ret = 0; - ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); - - if (unlikely(ret != 0)) { - if (ret != -ERESTARTSYS) { - pr_err("Failed to expire sync object before buffer eviction\n"); - } - goto out; - } - lockdep_assert_held(&bo->resv->lock.base); evict_mem = bo->mem; @@ -716,7 +707,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, no_wait_gpu); - if (ret) { + if (unlikely(ret)) { if (ret != -ERESTARTSYS) pr_err("Buffer eviction failed\n"); ttm_bo_mem_put(bo, &evict_mem); -- cgit v1.2.3 From 61ede070555395fb1c231311c35464d9cd1a8a81 Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 6 Jun 2016 10:17:57 +0200 Subject: drm/ttm: wait for BO idle after the move in ttm_bo_swapout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final part to avoid pre move waits. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 86dd695021a6..c3c615c525a4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1655,14 +1655,9 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) ttm_bo_list_ref_sub(bo, put_count, true); /** - * Wait for GPU, then move to system cached. + * Move to system cached */ - ret = ttm_bo_wait(bo, false, false); - - if (unlikely(ret != 0)) - goto out; - if ((bo->mem.placement & swap_placement) != swap_placement) { struct ttm_mem_reg evict_mem; @@ -1677,6 +1672,14 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) goto out; } + /** + * Make sure BO is idle. + */ + + ret = ttm_bo_wait(bo, false, false); + if (unlikely(ret != 0)) + goto out; + ttm_bo_unmap_virtual(bo); /** -- cgit v1.2.3 From 74561cd4f128091f41ab698277cde2542dcc5cad Mon Sep 17 00:00:00 2001 From: Christian König Date: Wed, 15 Jun 2016 13:44:00 +0200 Subject: drm/ttm: remove no_gpu_wait param from ttm_bo_move_accel_cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It isn't used and not waiting for the GPU after scheduling a move is actually quite dangerous. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 3 +-- drivers/gpu/drm/nouveau/nouveau_bo.c | 1 - drivers/gpu/drm/radeon/radeon_ttm.c | 3 +-- drivers/gpu/drm/ttm/ttm_bo_util.c | 1 - include/drm/ttm/ttm_bo_driver.h | 4 +--- 5 files changed, 3 insertions(+), 9 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 232f12344f79..b2b9df6686c8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -287,8 +287,7 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo, new_mem->num_pages * PAGE_SIZE, /* bytes */ bo->resv, &fence); /* FIXME: handle copy error */ - r = ttm_bo_move_accel_cleanup(bo, fence, - evict, no_wait_gpu, new_mem); + r = ttm_bo_move_accel_cleanup(bo, fence, evict, new_mem); fence_put(fence); return r; } diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 4fb1bf9b81b3..cdd551ef5bc8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -1082,7 +1082,6 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr, ret = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, - no_wait_gpu, new_mem); nouveau_fence_unref(&fence); } diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index 20ca22dc6813..ffdad81ef964 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -300,8 +300,7 @@ static int radeon_move_blit(struct ttm_buffer_object *bo, if (IS_ERR(fence)) return PTR_ERR(fence); - r = ttm_bo_move_accel_cleanup(bo, &fence->base, - evict, no_wait_gpu, new_mem); + r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, new_mem); radeon_fence_unref(&fence); return r; } diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 434f2394ad2c..c8fe554ee1c7 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -637,7 +637,6 @@ EXPORT_SYMBOL(ttm_bo_kunmap); int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, struct fence *fence, bool evict, - bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct ttm_bo_device *bdev = bo->bdev; diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 0d1d9d78c650..697e5f919135 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -1004,7 +1004,6 @@ extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo); * @bo: A pointer to a struct ttm_buffer_object. * @fence: A fence object that signals when moving is complete. * @evict: This is an evict move. Don't return until the buffer is idle. - * @no_wait_gpu: Return immediately if the GPU is busy. * @new_mem: struct ttm_mem_reg indicating where to move. * * Accelerated move function to be called when an accelerated move @@ -1016,8 +1015,7 @@ extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo); */ extern int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, - struct fence *fence, - bool evict, bool no_wait_gpu, + struct fence *fence, bool evict, struct ttm_mem_reg *new_mem); /** * ttm_io_prot -- cgit v1.2.3 From 5bc730677b1698c479f0134926b90789759b17ee Mon Sep 17 00:00:00 2001 From: Christian König Date: Wed, 15 Jun 2016 13:44:01 +0200 Subject: drm/ttm: remove TTM_BO_PRIV_FLAG_MOVING MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of using the flag just remember the fence of the last move operation. This avoids waiting for command submissions pipelined after the move, but before accessing the BO with the CPU again. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 4 ++-- drivers/gpu/drm/ttm/ttm_bo_util.c | 4 +++- drivers/gpu/drm/ttm/ttm_bo_vm.c | 19 ++++++++++++------- include/drm/ttm/ttm_bo_api.h | 4 ++-- include/drm/ttm/ttm_bo_driver.h | 3 --- 5 files changed, 19 insertions(+), 15 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index c3c615c525a4..caa657d31ce3 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -149,6 +149,7 @@ static void ttm_bo_release_list(struct kref *list_kref) ttm_tt_destroy(bo->ttm); atomic_dec(&bo->glob->bo_count); + fence_put(bo->moving); if (bo->resv == &bo->ttm_resv) reservation_object_fini(&bo->ttm_resv); mutex_destroy(&bo->wu_mutex); @@ -1138,7 +1139,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev, bo->mem.page_alignment = page_alignment; bo->mem.bus.io_reserved_vm = false; bo->mem.bus.io_reserved_count = 0; - bo->priv_flags = 0; + bo->moving = NULL; bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); bo->persistent_swap_storage = persistent_swap_storage; bo->acc_size = acc_size; @@ -1585,7 +1586,6 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, return -EBUSY; reservation_object_add_excl_fence(resv, NULL); - clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); return 0; } EXPORT_SYMBOL(ttm_bo_wait); diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index c8fe554ee1c7..9ea8d02f4ea5 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -465,6 +465,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, INIT_LIST_HEAD(&fbo->lru); INIT_LIST_HEAD(&fbo->swap); INIT_LIST_HEAD(&fbo->io_reserve_lru); + fbo->moving = NULL; drm_vma_node_reset(&fbo->vma_node); atomic_set(&fbo->cpu_writers, 0); @@ -665,7 +666,8 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, * operation has completed. */ - set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); + fence_put(bo->moving); + bo->moving = fence_get(fence); ret = ttm_buffer_object_transfer(bo, &ghost_obj); if (ret) diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 3216878bced3..a6ed9d5e5167 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -48,15 +48,14 @@ static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, { int ret = 0; - if (likely(!test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags))) + if (likely(!bo->moving)) goto out_unlock; /* * Quick non-stalling check for idle. */ - ret = ttm_bo_wait(bo, false, true); - if (likely(ret == 0)) - goto out_unlock; + if (fence_is_signaled(bo->moving)) + goto out_clear; /* * If possible, avoid waiting for GPU with mmap_sem @@ -68,17 +67,23 @@ static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, goto out_unlock; up_read(&vma->vm_mm->mmap_sem); - (void) ttm_bo_wait(bo, true, false); + (void) fence_wait(bo->moving, true); goto out_unlock; } /* * Ordinary wait. */ - ret = ttm_bo_wait(bo, true, false); - if (unlikely(ret != 0)) + ret = fence_wait(bo->moving, true); + if (unlikely(ret != 0)) { ret = (ret != -ERESTARTSYS) ? VM_FAULT_SIGBUS : VM_FAULT_NOPAGE; + goto out_unlock; + } + +out_clear: + fence_put(bo->moving); + bo->moving = NULL; out_unlock: return ret; diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index c801d9028e37..97aaf5c3d45b 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -173,7 +173,7 @@ struct ttm_tt; * @lru: List head for the lru list. * @ddestroy: List head for the delayed destroy list. * @swap: List head for swap LRU list. - * @priv_flags: Flags describing buffer object internal state. + * @moving: Fence set when BO is moving * @vma_node: Address space manager node. * @offset: The current GPU offset, which can have different meanings * depending on the memory type. For SYSTEM type memory, it should be 0. @@ -239,7 +239,7 @@ struct ttm_buffer_object { * Members protected by a bo reservation. */ - unsigned long priv_flags; + struct fence *moving; struct drm_vma_offset_node vma_node; diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 697e5f919135..44dea22d594e 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -503,9 +503,6 @@ struct ttm_bo_global { #define TTM_NUM_MEM_TYPES 8 -#define TTM_BO_PRIV_FLAG_MOVING 0 /* Buffer object is moving and needs - idling before CPU mapping */ -#define TTM_BO_PRIV_FLAG_MAX 1 /** * struct ttm_bo_device - Buffer object driver device-specific data. * -- cgit v1.2.3 From f849c6d6527a4108b1d7813e1a0eac9f1568cc4b Mon Sep 17 00:00:00 2001 From: Christian König Date: Wed, 15 Jun 2016 13:44:02 +0200 Subject: drm/ttm: simplify ttm_bo_wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As far as I can see no need for a custom implementation any more. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 37 ++++--------------------------------- 1 file changed, 4 insertions(+), 33 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index caa657d31ce3..28cd5352f8d0 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1546,46 +1546,17 @@ EXPORT_SYMBOL(ttm_bo_unmap_virtual); int ttm_bo_wait(struct ttm_buffer_object *bo, bool interruptible, bool no_wait) { - struct reservation_object_list *fobj; - struct reservation_object *resv; - struct fence *excl; - long timeout = 15 * HZ; - int i; - - resv = bo->resv; - fobj = reservation_object_get_list(resv); - excl = reservation_object_get_excl(resv); - if (excl) { - if (!fence_is_signaled(excl)) { - if (no_wait) - return -EBUSY; - - timeout = fence_wait_timeout(excl, - interruptible, timeout); - } - } - - for (i = 0; fobj && timeout > 0 && i < fobj->shared_count; ++i) { - struct fence *fence; - fence = rcu_dereference_protected(fobj->shared[i], - reservation_object_held(resv)); - - if (!fence_is_signaled(fence)) { - if (no_wait) - return -EBUSY; - - timeout = fence_wait_timeout(fence, - interruptible, timeout); - } - } + long timeout = no_wait ? 0 : 15 * HZ; + timeout = reservation_object_wait_timeout_rcu(bo->resv, true, + interruptible, timeout); if (timeout < 0) return timeout; if (timeout == 0) return -EBUSY; - reservation_object_add_excl_fence(resv, NULL); + reservation_object_add_excl_fence(bo->resv, NULL); return 0; } EXPORT_SYMBOL(ttm_bo_wait); -- cgit v1.2.3 From 3ddf4ad9179779693a7656e67872fc37cc49e92b Mon Sep 17 00:00:00 2001 From: Christian König Date: Wed, 15 Jun 2016 13:44:03 +0200 Subject: drm/ttm: add the infrastructure for pipelined evictions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Free up the memory immediately, remember the last eviction for each domain and make new allocations depend on the last eviction to be completed. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 49 ++++++++++++++++++--- drivers/gpu/drm/ttm/ttm_bo_util.c | 92 +++++++++++++++++++++++++++++++++++++++ include/drm/ttm/ttm_bo_driver.h | 24 ++++++++++ 3 files changed, 160 insertions(+), 5 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 28cd5352f8d0..5d931690e0e1 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -787,6 +787,34 @@ void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) } EXPORT_SYMBOL(ttm_bo_mem_put); +/** + * Add the last move fence to the BO and reserve a new shared slot. + */ +static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, + struct ttm_mem_type_manager *man, + struct ttm_mem_reg *mem) +{ + struct fence *fence; + int ret; + + spin_lock(&man->move_lock); + fence = fence_get(man->move); + spin_unlock(&man->move_lock); + + if (fence) { + reservation_object_add_shared_fence(bo->resv, fence); + + ret = reservation_object_reserve_shared(bo->resv); + if (unlikely(ret)) + return ret; + + fence_put(bo->moving); + bo->moving = fence; + } + + return 0; +} + /** * Repeatedly evict memory from the LRU for @mem_type until we create enough * space, or we've evicted everything and there isn't enough space. @@ -813,10 +841,8 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, if (unlikely(ret != 0)) return ret; } while (1); - if (mem->mm_node == NULL) - return -ENOMEM; mem->mem_type = mem_type; - return 0; + return ttm_bo_add_move_fence(bo, man, mem); } static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, @@ -886,6 +912,10 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, bool has_erestartsys = false; int i, ret; + ret = reservation_object_reserve_shared(bo->resv); + if (unlikely(ret)) + return ret; + mem->mm_node = NULL; for (i = 0; i < placement->num_placement; ++i) { const struct ttm_place *place = &placement->placement[i]; @@ -919,9 +949,15 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, ret = (*man->func->get_node)(man, bo, place, mem); if (unlikely(ret)) return ret; - - if (mem->mm_node) + + if (mem->mm_node) { + ret = ttm_bo_add_move_fence(bo, man, mem); + if (unlikely(ret)) { + (*man->func->put_node)(man, mem); + return ret; + } break; + } } if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { @@ -1290,6 +1326,7 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) mem_type); return ret; } + fence_put(man->move); man->use_type = false; man->has_type = false; @@ -1335,6 +1372,7 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, man->io_reserve_fastpath = true; man->use_io_reserve_lru = false; mutex_init(&man->io_reserve_mutex); + spin_lock_init(&man->move_lock); INIT_LIST_HEAD(&man->io_reserve_lru); ret = bdev->driver->init_mem_type(bdev, type, man); @@ -1353,6 +1391,7 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, man->size = p_size; INIT_LIST_HEAD(&man->lru); + man->move = NULL; return 0; } diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 9ea8d02f4ea5..0c389a54cac1 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -696,3 +696,95 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, return 0; } EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); + +int ttm_bo_pipeline_move(struct ttm_buffer_object *bo, + struct fence *fence, bool evict, + struct ttm_mem_reg *new_mem) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_reg *old_mem = &bo->mem; + + struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type]; + struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type]; + + int ret; + + reservation_object_add_excl_fence(bo->resv, fence); + + if (!evict) { + struct ttm_buffer_object *ghost_obj; + + /** + * This should help pipeline ordinary buffer moves. + * + * Hang old buffer memory on a new buffer object, + * and leave it to be released when the GPU + * operation has completed. + */ + + fence_put(bo->moving); + bo->moving = fence_get(fence); + + ret = ttm_buffer_object_transfer(bo, &ghost_obj); + if (ret) + return ret; + + reservation_object_add_excl_fence(ghost_obj->resv, fence); + + /** + * If we're not moving to fixed memory, the TTM object + * needs to stay alive. Otherwhise hang it on the ghost + * bo to be unbound and destroyed. + */ + + if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED)) + ghost_obj->ttm = NULL; + else + bo->ttm = NULL; + + ttm_bo_unreserve(ghost_obj); + ttm_bo_unref(&ghost_obj); + + } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) { + + /** + * BO doesn't have a TTM we need to bind/unbind. Just remember + * this eviction and free up the allocation + */ + + spin_lock(&from->move_lock); + if (!from->move || fence_is_later(from->move, fence)) { + fence_put(from->move); + from->move = fence_get(fence); + } + spin_unlock(&from->move_lock); + + ttm_bo_free_old_node(bo); + + fence_put(bo->moving); + bo->moving = fence_get(fence); + + } else { + /** + * Last resort, wait for the move to be completed. + * + * Should never happen in pratice. + */ + + ret = ttm_bo_wait(bo, false, false); + if (ret) + return ret; + + if (to->flags & TTM_MEMTYPE_FLAG_FIXED) { + ttm_tt_destroy(bo->ttm); + bo->ttm = NULL; + } + ttm_bo_free_old_node(bo); + } + + *old_mem = *new_mem; + new_mem->mm_node = NULL; + + return 0; +} +EXPORT_SYMBOL(ttm_bo_pipeline_move); diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 44dea22d594e..e2ebe6666e2b 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -258,8 +258,10 @@ struct ttm_mem_type_manager_func { * reserved by the TTM vm system. * @io_reserve_lru: Optional lru list for unreserving io mem regions. * @io_reserve_fastpath: Only use bdev::driver::io_mem_reserve to obtain + * @move_lock: lock for move fence * static information. bdev::driver::io_mem_free is never used. * @lru: The lru list for this memory type. + * @move: The fence of the last pipelined move operation. * * This structure is used to identify and manage memory types for a device. * It's set up by the ttm_bo_driver::init_mem_type method. @@ -286,6 +288,7 @@ struct ttm_mem_type_manager { struct mutex io_reserve_mutex; bool use_io_reserve_lru; bool io_reserve_fastpath; + spinlock_t move_lock; /* * Protected by @io_reserve_mutex: @@ -298,6 +301,11 @@ struct ttm_mem_type_manager { */ struct list_head lru; + + /* + * Protected by @move_lock. + */ + struct fence *move; }; /** @@ -1014,6 +1022,22 @@ extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo); extern int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, struct fence *fence, bool evict, struct ttm_mem_reg *new_mem); + +/** + * ttm_bo_pipeline_move. + * + * @bo: A pointer to a struct ttm_buffer_object. + * @fence: A fence object that signals when moving is complete. + * @evict: This is an evict move. Don't return until the buffer is idle. + * @new_mem: struct ttm_mem_reg indicating where to move. + * + * Function for pipelining accelerated moves. Either free the memory + * immediately or hang it on a temporary buffer object. + */ +int ttm_bo_pipeline_move(struct ttm_buffer_object *bo, + struct fence *fence, bool evict, + struct ttm_mem_reg *new_mem); + /** * ttm_io_prot * -- cgit v1.2.3 From aff98ba1fdb8150a1a7afe61503772f61b9a2132 Mon Sep 17 00:00:00 2001 From: Christian König Date: Wed, 22 Jun 2016 14:16:28 +0200 Subject: drm/ttm: wait for eviction in ttm_bo_force_list_clean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we can pipeline evictions we need to wait for them to finish when we cleanup a memory domain. Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 5d931690e0e1..e340d0d66429 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1287,6 +1287,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, { struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_bo_global *glob = bdev->glob; + struct fence *fence; int ret; /* @@ -1307,6 +1308,23 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, spin_lock(&glob->lru_lock); } spin_unlock(&glob->lru_lock); + + spin_lock(&man->move_lock); + fence = fence_get(man->move); + spin_unlock(&man->move_lock); + + if (fence) { + ret = fence_wait(fence, false); + fence_put(fence); + if (ret) { + if (allow_errors) { + return ret; + } else { + pr_err("Cleanup eviction failed\n"); + } + } + } + return 0; } -- cgit v1.2.3 From 847b6cf9fb771341a550661ef61d510722eba785 Mon Sep 17 00:00:00 2001 From: Christian König Date: Fri, 24 Jun 2016 21:51:03 +0200 Subject: drm/ttm: fix stupid parameter inversion in the pipeline code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want to keep the newest fence, not the oldest one. Reviewed-by: Alex Deucher Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 0c389a54cac1..4da0e784f9e7 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -753,7 +753,7 @@ int ttm_bo_pipeline_move(struct ttm_buffer_object *bo, */ spin_lock(&from->move_lock); - if (!from->move || fence_is_later(from->move, fence)) { + if (!from->move || fence_is_later(fence, from->move)) { fence_put(from->move); from->move = fence_get(fence); } -- cgit v1.2.3