summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/gt/intel_ring_submission.c
diff options
context:
space:
mode:
authorMaxime Ripard <maxime@cerno.tech>2021-05-11 13:35:52 +0200
committerMaxime Ripard <maxime@cerno.tech>2021-05-11 13:35:52 +0200
commitc55b44c9386f3ee1b08752638559f19deaf6040d (patch)
treec843a21f45180387fcd9eb2625cc9d1f166a3156 /drivers/gpu/drm/i915/gt/intel_ring_submission.c
parentMAINTAINERS: Update my e-mail (diff)
parentLinux 5.13-rc1 (diff)
downloadlinux-c55b44c9386f3ee1b08752638559f19deaf6040d.tar.xz
linux-c55b44c9386f3ee1b08752638559f19deaf6040d.zip
Merge drm/drm-fixes into drm-misc-fixes
Start this new release drm-misc-fixes branch Signed-off-by: Maxime Ripard <maxime@cerno.tech>
Diffstat (limited to 'drivers/gpu/drm/i915/gt/intel_ring_submission.c')
-rw-r--r--drivers/gpu/drm/i915/gt/intel_ring_submission.c330
1 files changed, 182 insertions, 148 deletions
diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
index 4984ff565424..9585546556ee 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c
@@ -1,30 +1,6 @@
+// SPDX-License-Identifier: MIT
/*
- * Copyright © 2008-2010 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- *
- * Authors:
- * Eric Anholt <eric@anholt.net>
- * Zou Nan hai <nanhai.zou@intel.com>
- * Xiang Hai hao<haihao.xiang@intel.com>
- *
+ * Copyright © 2008-2021 Intel Corporation
*/
#include "gen2_engine_cs.h"
@@ -183,15 +159,36 @@ static void set_pp_dir(struct intel_engine_cs *engine)
}
}
+static bool stop_ring(struct intel_engine_cs *engine)
+{
+ /* Empty the ring by skipping to the end */
+ ENGINE_WRITE_FW(engine, RING_HEAD, ENGINE_READ_FW(engine, RING_TAIL));
+ ENGINE_POSTING_READ(engine, RING_HEAD);
+
+ /* The ring must be empty before it is disabled */
+ ENGINE_WRITE_FW(engine, RING_CTL, 0);
+ ENGINE_POSTING_READ(engine, RING_CTL);
+
+ /* Then reset the disabled ring */
+ ENGINE_WRITE_FW(engine, RING_HEAD, 0);
+ ENGINE_WRITE_FW(engine, RING_TAIL, 0);
+
+ return (ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) == 0;
+}
+
static int xcs_resume(struct intel_engine_cs *engine)
{
- struct drm_i915_private *dev_priv = engine->i915;
struct intel_ring *ring = engine->legacy.ring;
ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n",
ring->head, ring->tail);
- if (HWS_NEEDS_PHYSICAL(dev_priv))
+ /* Double check the ring is empty & disabled before we resume */
+ synchronize_hardirq(engine->i915->drm.irq);
+ if (!stop_ring(engine))
+ goto err;
+
+ if (HWS_NEEDS_PHYSICAL(engine->i915))
ring_setup_phys_status_page(engine);
else
ring_setup_status_page(engine);
@@ -228,21 +225,10 @@ static int xcs_resume(struct intel_engine_cs *engine)
if (__intel_wait_for_register_fw(engine->uncore,
RING_CTL(engine->mmio_base),
RING_VALID, RING_VALID,
- 5000, 0, NULL)) {
- drm_err(&dev_priv->drm,
- "%s initialization failed; "
- "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
- engine->name,
- ENGINE_READ(engine, RING_CTL),
- ENGINE_READ(engine, RING_CTL) & RING_VALID,
- ENGINE_READ(engine, RING_HEAD), ring->head,
- ENGINE_READ(engine, RING_TAIL), ring->tail,
- ENGINE_READ(engine, RING_START),
- i915_ggtt_offset(ring->vma));
- return -EIO;
- }
+ 5000, 0, NULL))
+ goto err;
- if (INTEL_GEN(dev_priv) > 2)
+ if (INTEL_GEN(engine->i915) > 2)
ENGINE_WRITE_FW(engine,
RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
@@ -255,6 +241,19 @@ static int xcs_resume(struct intel_engine_cs *engine)
/* Papering over lost _interrupts_ immediately following the restart */
intel_engine_signal_breadcrumbs(engine);
return 0;
+
+err:
+ drm_err(&engine->i915->drm,
+ "%s initialization failed; "
+ "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
+ engine->name,
+ ENGINE_READ(engine, RING_CTL),
+ ENGINE_READ(engine, RING_CTL) & RING_VALID,
+ ENGINE_READ(engine, RING_HEAD), ring->head,
+ ENGINE_READ(engine, RING_TAIL), ring->tail,
+ ENGINE_READ(engine, RING_START),
+ i915_ggtt_offset(ring->vma));
+ return -EIO;
}
static void sanitize_hwsp(struct intel_engine_cs *engine)
@@ -290,23 +289,6 @@ static void xcs_sanitize(struct intel_engine_cs *engine)
clflush_cache_range(engine->status_page.addr, PAGE_SIZE);
}
-static bool stop_ring(struct intel_engine_cs *engine)
-{
- /* Empty the ring by skipping to the end */
- ENGINE_WRITE_FW(engine, RING_HEAD, ENGINE_READ_FW(engine, RING_TAIL));
- ENGINE_POSTING_READ(engine, RING_HEAD);
-
- /* The ring must be empty before it is disabled */
- ENGINE_WRITE_FW(engine, RING_CTL, 0);
- ENGINE_POSTING_READ(engine, RING_CTL);
-
- /* Then reset the disabled ring */
- ENGINE_WRITE_FW(engine, RING_HEAD, 0);
- ENGINE_WRITE_FW(engine, RING_TAIL, 0);
-
- return (ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) == 0;
-}
-
static void reset_prepare(struct intel_engine_cs *engine)
{
/*
@@ -329,25 +311,23 @@ static void reset_prepare(struct intel_engine_cs *engine)
if (!stop_ring(engine)) {
/* G45 ring initialization often fails to reset head to zero */
- drm_dbg(&engine->i915->drm,
- "%s head not reset to zero "
- "ctl %08x head %08x tail %08x start %08x\n",
- engine->name,
- ENGINE_READ_FW(engine, RING_CTL),
- ENGINE_READ_FW(engine, RING_HEAD),
- ENGINE_READ_FW(engine, RING_TAIL),
- ENGINE_READ_FW(engine, RING_START));
- }
-
- if (!stop_ring(engine)) {
- drm_err(&engine->i915->drm,
- "failed to set %s head to zero "
- "ctl %08x head %08x tail %08x start %08x\n",
- engine->name,
- ENGINE_READ_FW(engine, RING_CTL),
- ENGINE_READ_FW(engine, RING_HEAD),
- ENGINE_READ_FW(engine, RING_TAIL),
- ENGINE_READ_FW(engine, RING_START));
+ ENGINE_TRACE(engine,
+ "HEAD not reset to zero, "
+ "{ CTL:%08x, HEAD:%08x, TAIL:%08x, START:%08x }\n",
+ ENGINE_READ_FW(engine, RING_CTL),
+ ENGINE_READ_FW(engine, RING_HEAD),
+ ENGINE_READ_FW(engine, RING_TAIL),
+ ENGINE_READ_FW(engine, RING_START));
+ if (!stop_ring(engine)) {
+ drm_err(&engine->i915->drm,
+ "failed to set %s head to zero "
+ "ctl %08x head %08x tail %08x start %08x\n",
+ engine->name,
+ ENGINE_READ_FW(engine, RING_CTL),
+ ENGINE_READ_FW(engine, RING_HEAD),
+ ENGINE_READ_FW(engine, RING_TAIL),
+ ENGINE_READ_FW(engine, RING_START));
+ }
}
}
@@ -431,7 +411,7 @@ static void reset_cancel(struct intel_engine_cs *engine)
/* Mark all submitted requests as skipped. */
list_for_each_entry(request, &engine->active.requests, sched.link)
- i915_request_mark_eio(request);
+ i915_request_put(i915_request_mark_eio(request));
intel_engine_signal_breadcrumbs(engine);
/* Remaining _unready_ requests will be nop'ed when submitted */
@@ -466,6 +446,26 @@ static void ring_context_destroy(struct kref *ref)
intel_context_free(ce);
}
+static int ring_context_init_default_state(struct intel_context *ce,
+ struct i915_gem_ww_ctx *ww)
+{
+ struct drm_i915_gem_object *obj = ce->state->obj;
+ void *vaddr;
+
+ vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+ if (IS_ERR(vaddr))
+ return PTR_ERR(vaddr);
+
+ shmem_read(ce->engine->default_state, 0,
+ vaddr, ce->engine->context_size);
+
+ i915_gem_object_flush_map(obj);
+ __i915_gem_object_release_map(obj);
+
+ __set_bit(CONTEXT_VALID_BIT, &ce->flags);
+ return 0;
+}
+
static int ring_context_pre_pin(struct intel_context *ce,
struct i915_gem_ww_ctx *ww,
void **unused)
@@ -473,6 +473,13 @@ static int ring_context_pre_pin(struct intel_context *ce,
struct i915_address_space *vm;
int err = 0;
+ if (ce->engine->default_state &&
+ !test_bit(CONTEXT_VALID_BIT, &ce->flags)) {
+ err = ring_context_init_default_state(ce, ww);
+ if (err)
+ return err;
+ }
+
vm = vm_alias(ce->vm);
if (vm)
err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww);
@@ -528,22 +535,6 @@ alloc_context_vma(struct intel_engine_cs *engine)
if (IS_IVYBRIDGE(i915))
i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC);
- if (engine->default_state) {
- void *vaddr;
-
- vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
- if (IS_ERR(vaddr)) {
- err = PTR_ERR(vaddr);
- goto err_obj;
- }
-
- shmem_read(engine->default_state, 0,
- vaddr, engine->context_size);
-
- i915_gem_object_flush_map(obj);
- __i915_gem_object_release_map(obj);
- }
-
vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
if (IS_ERR(vma)) {
err = PTR_ERR(vma);
@@ -575,8 +566,6 @@ static int ring_context_alloc(struct intel_context *ce)
return PTR_ERR(vma);
ce->state = vma;
- if (engine->default_state)
- __set_bit(CONTEXT_VALID_BIT, &ce->flags);
}
return 0;
@@ -761,13 +750,14 @@ static int mi_set_context(struct i915_request *rq,
static int remap_l3_slice(struct i915_request *rq, int slice)
{
+#define L3LOG_DW (GEN7_L3LOG_SIZE / sizeof(u32))
u32 *cs, *remap_info = rq->engine->i915->l3_parity.remap_info[slice];
int i;
if (!remap_info)
return 0;
- cs = intel_ring_begin(rq, GEN7_L3LOG_SIZE/4 * 2 + 2);
+ cs = intel_ring_begin(rq, L3LOG_DW * 2 + 2);
if (IS_ERR(cs))
return PTR_ERR(cs);
@@ -776,8 +766,8 @@ static int remap_l3_slice(struct i915_request *rq, int slice)
* here because no other code should access these registers other than
* at initialization time.
*/
- *cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4);
- for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) {
+ *cs++ = MI_LOAD_REGISTER_IMM(L3LOG_DW);
+ for (i = 0; i < L3LOG_DW; i++) {
*cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i));
*cs++ = remap_info[i];
}
@@ -785,6 +775,7 @@ static int remap_l3_slice(struct i915_request *rq, int slice)
intel_ring_advance(rq, cs);
return 0;
+#undef L3LOG_DW
}
static int remap_l3(struct i915_request *rq)
@@ -1176,37 +1167,15 @@ static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine,
return gen7_setup_clear_gpr_bb(engine, vma);
}
-static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine)
+static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine,
+ struct i915_gem_ww_ctx *ww,
+ struct i915_vma *vma)
{
- struct drm_i915_gem_object *obj;
- struct i915_vma *vma;
- int size;
int err;
- size = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */);
- if (size <= 0)
- return size;
-
- size = ALIGN(size, PAGE_SIZE);
- obj = i915_gem_object_create_internal(engine->i915, size);
- if (IS_ERR(obj))
- return PTR_ERR(obj);
-
- vma = i915_vma_instance(obj, engine->gt->vm, NULL);
- if (IS_ERR(vma)) {
- err = PTR_ERR(vma);
- goto err_obj;
- }
-
- vma->private = intel_context_create(engine); /* dummy residuals */
- if (IS_ERR(vma->private)) {
- err = PTR_ERR(vma->private);
- goto err_obj;
- }
-
- err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH);
+ err = i915_vma_pin_ww(vma, ww, 0, 0, PIN_USER | PIN_HIGH);
if (err)
- goto err_private;
+ return err;
err = i915_vma_sync(vma);
if (err)
@@ -1221,17 +1190,53 @@ static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine)
err_unpin:
i915_vma_unpin(vma);
-err_private:
- intel_context_put(vma->private);
-err_obj:
- i915_gem_object_put(obj);
return err;
}
+static struct i915_vma *gen7_ctx_vma(struct intel_engine_cs *engine)
+{
+ struct drm_i915_gem_object *obj;
+ struct i915_vma *vma;
+ int size, err;
+
+ if (!IS_GEN(engine->i915, 7) || engine->class != RENDER_CLASS)
+ return 0;
+
+ err = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */);
+ if (err < 0)
+ return ERR_PTR(err);
+ if (!err)
+ return NULL;
+
+ size = ALIGN(err, PAGE_SIZE);
+
+ obj = i915_gem_object_create_internal(engine->i915, size);
+ if (IS_ERR(obj))
+ return ERR_CAST(obj);
+
+ vma = i915_vma_instance(obj, engine->gt->vm, NULL);
+ if (IS_ERR(vma)) {
+ i915_gem_object_put(obj);
+ return ERR_CAST(vma);
+ }
+
+ vma->private = intel_context_create(engine); /* dummy residuals */
+ if (IS_ERR(vma->private)) {
+ err = PTR_ERR(vma->private);
+ vma->private = NULL;
+ i915_gem_object_put(obj);
+ return ERR_PTR(err);
+ }
+
+ return vma;
+}
+
int intel_ring_submission_setup(struct intel_engine_cs *engine)
{
+ struct i915_gem_ww_ctx ww;
struct intel_timeline *timeline;
struct intel_ring *ring;
+ struct i915_vma *gen7_wa_vma;
int err;
setup_common(engine);
@@ -1262,43 +1267,72 @@ int intel_ring_submission_setup(struct intel_engine_cs *engine)
}
GEM_BUG_ON(timeline->has_initial_breadcrumb);
- err = intel_timeline_pin(timeline, NULL);
- if (err)
- goto err_timeline;
-
ring = intel_engine_create_ring(engine, SZ_16K);
if (IS_ERR(ring)) {
err = PTR_ERR(ring);
- goto err_timeline_unpin;
+ goto err_timeline;
}
- err = intel_ring_pin(ring, NULL);
- if (err)
- goto err_ring;
-
GEM_BUG_ON(engine->legacy.ring);
engine->legacy.ring = ring;
engine->legacy.timeline = timeline;
- GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);
+ gen7_wa_vma = gen7_ctx_vma(engine);
+ if (IS_ERR(gen7_wa_vma)) {
+ err = PTR_ERR(gen7_wa_vma);
+ goto err_ring;
+ }
- if (IS_GEN(engine->i915, 7) && engine->class == RENDER_CLASS) {
- err = gen7_ctx_switch_bb_init(engine);
+ i915_gem_ww_ctx_init(&ww, false);
+
+retry:
+ err = i915_gem_object_lock(timeline->hwsp_ggtt->obj, &ww);
+ if (!err && gen7_wa_vma)
+ err = i915_gem_object_lock(gen7_wa_vma->obj, &ww);
+ if (!err && engine->legacy.ring->vma->obj)
+ err = i915_gem_object_lock(engine->legacy.ring->vma->obj, &ww);
+ if (!err)
+ err = intel_timeline_pin(timeline, &ww);
+ if (!err) {
+ err = intel_ring_pin(ring, &ww);
if (err)
- goto err_ring_unpin;
+ intel_timeline_unpin(timeline);
+ }
+ if (err)
+ goto out;
+
+ GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);
+
+ if (gen7_wa_vma) {
+ err = gen7_ctx_switch_bb_init(engine, &ww, gen7_wa_vma);
+ if (err) {
+ intel_ring_unpin(ring);
+ intel_timeline_unpin(timeline);
+ }
}
+out:
+ if (err == -EDEADLK) {
+ err = i915_gem_ww_ctx_backoff(&ww);
+ if (!err)
+ goto retry;
+ }
+ i915_gem_ww_ctx_fini(&ww);
+ if (err)
+ goto err_gen7_put;
+
/* Finally, take ownership and responsibility for cleanup! */
engine->release = ring_release;
return 0;
-err_ring_unpin:
- intel_ring_unpin(ring);
+err_gen7_put:
+ if (gen7_wa_vma) {
+ intel_context_put(gen7_wa_vma->private);
+ i915_gem_object_put(gen7_wa_vma->obj);
+ }
err_ring:
intel_ring_put(ring);
-err_timeline_unpin:
- intel_timeline_unpin(timeline);
err_timeline:
intel_timeline_put(timeline);
err: