diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index 2bda426a6986..d1561ebe4e56 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -313,6 +313,8 @@ static void dev_fini_ggtt(void *arg) { struct xe_ggtt *ggtt = arg; + scoped_guard(mutex, &ggtt->lock) + ggtt->flags &= ~XE_GGTT_FLAGS_ONLINE; drain_workqueue(ggtt->wq); } @@ -377,6 +379,7 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt) if (err) return err; + ggtt->flags |= XE_GGTT_FLAGS_ONLINE; err = devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt); if (err) return err; @@ -410,13 +413,10 @@ static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt) static void ggtt_node_remove(struct xe_ggtt_node *node) { struct xe_ggtt *ggtt = node->ggtt; - struct xe_device *xe = tile_to_xe(ggtt->tile); bool bound; - int idx; - - bound = drm_dev_enter(&xe->drm, &idx); mutex_lock(&ggtt->lock); + bound = ggtt->flags & XE_GGTT_FLAGS_ONLINE; if (bound) xe_ggtt_clear(ggtt, node->base.start, node->base.size); drm_mm_remove_node(&node->base); @@ -429,8 +429,6 @@ static void ggtt_node_remove(struct xe_ggtt_node *node) if (node->invalidate_on_remove) xe_ggtt_invalidate(ggtt); - drm_dev_exit(idx); - free_node: xe_ggtt_node_fini(node); } diff --git a/drivers/gpu/drm/xe/xe_ggtt_types.h b/drivers/gpu/drm/xe/xe_ggtt_types.h index d82b71a198bc..c002857bb761 100644 --- a/drivers/gpu/drm/xe/xe_ggtt_types.h +++ b/drivers/gpu/drm/xe/xe_ggtt_types.h @@ -28,11 +28,14 @@ struct xe_ggtt { /** @size: Total usable size of this GGTT */ u64 size; -#define XE_GGTT_FLAGS_64K BIT(0) +#define XE_GGTT_FLAGS_64K BIT(0) +#define XE_GGTT_FLAGS_ONLINE BIT(1) /** * @flags: Flags for this GGTT * Acceptable flags: * - %XE_GGTT_FLAGS_64K - if PTE size is 64K. Otherwise, regular is 4K. + * - %XE_GGTT_FLAGS_ONLINE - is GGTT online, protected by ggtt->lock + * after init */ unsigned int flags; /** @scratch: Internal object allocation used as a scratch page */ diff --git a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c index fe944687728c..03c1862ba497 100644 --- a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c +++ b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c @@ -12,6 +12,7 @@ #include "xe_gt_printk.h" #include "xe_gt_sysfs.h" #include "xe_mmio.h" +#include "xe_pm.h" #include "xe_sriov.h" static void __xe_gt_apply_ccs_mode(struct xe_gt *gt, u32 num_engines) @@ -150,6 +151,7 @@ ccs_mode_store(struct device *kdev, struct device_attribute *attr, xe_gt_info(gt, "Setting compute mode to %d\n", num_engines); gt->ccs_mode = num_engines; xe_gt_record_user_engines(gt); + guard(xe_pm_runtime)(xe); xe_gt_reset(gt); } diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index 6df7c3f260e5..4ab65cae8743 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -1124,14 +1124,14 @@ static int guc_wait_ucode(struct xe_guc *guc) struct xe_guc_pc *guc_pc = >->uc.guc.pc; u32 before_freq, act_freq, cur_freq; u32 status = 0, tries = 0; + int load_result, ret; ktime_t before; u64 delta_ms; - int ret; before_freq = xe_guc_pc_get_act_freq(guc_pc); before = ktime_get(); - ret = poll_timeout_us(ret = guc_load_done(gt, &status, &tries), ret, + ret = poll_timeout_us(load_result = guc_load_done(gt, &status, &tries), load_result, 10 * USEC_PER_MSEC, GUC_LOAD_TIMEOUT_SEC * USEC_PER_SEC, false); @@ -1139,7 +1139,7 @@ static int guc_wait_ucode(struct xe_guc *guc) act_freq = xe_guc_pc_get_act_freq(guc_pc); cur_freq = xe_guc_pc_get_cur_freq_fw(guc_pc); - if (ret) { + if (ret || load_result <= 0) { xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz)\n", status, delta_ms, xe_guc_pc_get_act_freq(guc_pc), xe_guc_pc_get_cur_freq_fw(guc_pc)); @@ -1347,15 +1347,37 @@ int xe_guc_enable_communication(struct xe_guc *guc) return 0; } -int xe_guc_suspend(struct xe_guc *guc) +/** + * xe_guc_softreset() - Soft reset GuC + * @guc: The GuC object + * + * Send soft reset command to GuC through mmio send. + * + * Return: 0 if success, otherwise error code + */ +int xe_guc_softreset(struct xe_guc *guc) { - struct xe_gt *gt = guc_to_gt(guc); u32 action[] = { XE_GUC_ACTION_CLIENT_SOFT_RESET, }; int ret; + if (!xe_uc_fw_is_running(&guc->fw)) + return 0; + ret = xe_guc_mmio_send(guc, action, ARRAY_SIZE(action)); + if (ret) + return ret; + + return 0; +} + +int xe_guc_suspend(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + int ret; + + ret = xe_guc_softreset(guc); if (ret) { xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret)); return ret; diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h index 66e7edc70ed9..02514914f404 100644 --- a/drivers/gpu/drm/xe/xe_guc.h +++ b/drivers/gpu/drm/xe/xe_guc.h @@ -44,6 +44,7 @@ int xe_guc_opt_in_features_enable(struct xe_guc *guc); void xe_guc_runtime_suspend(struct xe_guc *guc); void xe_guc_runtime_resume(struct xe_guc *guc); int xe_guc_suspend(struct xe_guc *guc); +int xe_guc_softreset(struct xe_guc *guc); void xe_guc_notify(struct xe_guc *guc); int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr); int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len); diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index d04589140b77..c80082b4c876 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -345,6 +345,7 @@ static void guc_action_disable_ct(void *arg) { struct xe_guc_ct *ct = arg; + xe_guc_ct_stop(ct); guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); } diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 799ef9f48003..fc4f99d46763 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -48,6 +48,8 @@ #define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN 6 +static int guc_submit_reset_prepare(struct xe_guc *guc); + static struct xe_guc * exec_queue_to_guc(struct xe_exec_queue *q) { @@ -239,7 +241,7 @@ static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) EXEC_QUEUE_STATE_BANNED)); } -static void guc_submit_fini(struct drm_device *drm, void *arg) +static void guc_submit_sw_fini(struct drm_device *drm, void *arg) { struct xe_guc *guc = arg; struct xe_device *xe = guc_to_xe(guc); @@ -257,6 +259,19 @@ static void guc_submit_fini(struct drm_device *drm, void *arg) xa_destroy(&guc->submission_state.exec_queue_lookup); } +static void guc_submit_fini(void *arg) +{ + struct xe_guc *guc = arg; + + /* Forcefully kill any remaining exec queues */ + xe_guc_ct_stop(&guc->ct); + guc_submit_reset_prepare(guc); + xe_guc_softreset(guc); + xe_guc_submit_stop(guc); + xe_uc_fw_sanitize(&guc->fw); + xe_guc_submit_pause_abort(guc); +} + static void guc_submit_wedged_fini(void *arg) { struct xe_guc *guc = arg; @@ -326,7 +341,11 @@ int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) guc->submission_state.initialized = true; - return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc); + err = drmm_add_action_or_reset(&xe->drm, guc_submit_sw_fini, guc); + if (err) + return err; + + return devm_add_action_or_reset(xe->drm.dev, guc_submit_fini, guc); } /* @@ -1252,6 +1271,7 @@ static void disable_scheduling_deregister(struct xe_guc *guc, */ void xe_guc_submit_wedge(struct xe_guc *guc) { + struct xe_device *xe = guc_to_xe(guc); struct xe_gt *gt = guc_to_gt(guc); struct xe_exec_queue *q; unsigned long index; @@ -1266,20 +1286,28 @@ void xe_guc_submit_wedge(struct xe_guc *guc) if (!guc->submission_state.initialized) return; - err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev, - guc_submit_wedged_fini, guc); - if (err) { - xe_gt_err(gt, "Failed to register clean-up in wedged.mode=%s; " - "Although device is wedged.\n", - xe_wedged_mode_to_string(XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET)); - return; - } + if (xe->wedged.mode == 2) { + err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev, + guc_submit_wedged_fini, guc); + if (err) { + xe_gt_err(gt, "Failed to register clean-up on wedged.mode=2; " + "Although device is wedged.\n"); + return; + } - mutex_lock(&guc->submission_state.lock); - xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) - if (xe_exec_queue_get_unless_zero(q)) - set_exec_queue_wedged(q); - mutex_unlock(&guc->submission_state.lock); + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + if (xe_exec_queue_get_unless_zero(q)) + set_exec_queue_wedged(q); + mutex_unlock(&guc->submission_state.lock); + } else { + /* Forcefully kill any remaining exec queues, signal fences */ + guc_submit_reset_prepare(guc); + xe_guc_submit_stop(guc); + xe_guc_softreset(guc); + xe_uc_fw_sanitize(&guc->fw); + xe_guc_submit_pause_abort(guc); + } } static bool guc_submit_hint_wedged(struct xe_guc *guc) @@ -2230,6 +2258,7 @@ static const struct xe_exec_queue_ops guc_exec_queue_ops = { static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; + bool do_destroy = false; /* Stop scheduling + flush any DRM scheduler operations */ xe_sched_submission_stop(sched); @@ -2237,7 +2266,7 @@ static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) /* Clean up lost G2H + reset engine state */ if (exec_queue_registered(q)) { if (exec_queue_destroyed(q)) - __guc_exec_queue_destroy(guc, q); + do_destroy = true; } if (q->guc->suspend_pending) { set_exec_queue_suspended(q); @@ -2273,18 +2302,15 @@ static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) xe_guc_exec_queue_trigger_cleanup(q); } } + + if (do_destroy) + __guc_exec_queue_destroy(guc, q); } -int xe_guc_submit_reset_prepare(struct xe_guc *guc) +static int guc_submit_reset_prepare(struct xe_guc *guc) { int ret; - if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc))) - return 0; - - if (!guc->submission_state.initialized) - return 0; - /* * Using an atomic here rather than submission_state.lock as this * function can be called while holding the CT lock (engine reset @@ -2299,6 +2325,17 @@ int xe_guc_submit_reset_prepare(struct xe_guc *guc) return ret; } +int xe_guc_submit_reset_prepare(struct xe_guc *guc) +{ + if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc))) + return 0; + + if (!guc->submission_state.initialized) + return 0; + + return guc_submit_reset_prepare(guc); +} + void xe_guc_submit_reset_wait(struct xe_guc *guc) { wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) || @@ -2695,8 +2732,7 @@ void xe_guc_submit_pause_abort(struct xe_guc *guc) continue; xe_sched_submission_start(sched); - if (exec_queue_killed_or_banned_or_wedged(q)) - xe_guc_exec_queue_trigger_cleanup(q); + guc_exec_queue_kill(q); } mutex_unlock(&guc->submission_state.lock); } diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index b0f037bc227f..7b70cc01fdb3 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -2413,14 +2413,14 @@ static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) * @lrc: Pointer to the lrc. * * Return latest ctx timestamp. With support for active contexts, the - * calculation may bb slightly racy, so follow a read-again logic to ensure that + * calculation may be slightly racy, so follow a read-again logic to ensure that * the context is still active before returning the right timestamp. * * Returns: New ctx timestamp value */ u64 xe_lrc_timestamp(struct xe_lrc *lrc) { - u64 lrc_ts, reg_ts, new_ts; + u64 lrc_ts, reg_ts, new_ts = lrc->ctx_timestamp; u32 engine_id; lrc_ts = xe_lrc_ctx_timestamp(lrc); diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 4dd3f29933cf..fa90441d3052 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -543,8 +543,7 @@ static ssize_t xe_oa_read(struct file *file, char __user *buf, size_t offset = 0; int ret; - /* Can't read from disabled streams */ - if (!stream->enabled || !stream->sample) + if (!stream->sample) return -EINVAL; if (!(file->f_flags & O_NONBLOCK)) { @@ -1460,6 +1459,10 @@ static void xe_oa_stream_disable(struct xe_oa_stream *stream) if (stream->sample) hrtimer_cancel(&stream->poll_check_timer); + + /* Update stream->oa_buffer.tail to allow any final reports to be read */ + if (xe_oa_buffer_check_unlocked(stream)) + wake_up(&stream->poll_wq); } static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream) diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 13b355fadd58..2d9ce2c4cb4f 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -1655,14 +1655,35 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, XE_WARN_ON(!level); /* Check for leaf node */ if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) && - (!xe_child->base.children || !xe_child->base.children[first])) { + xe_child->level <= MAX_HUGEPTE_LEVEL) { struct iosys_map *leaf_map = &xe_child->bo->vmap; pgoff_t count = xe_pt_num_entries(addr, next, xe_child->level, walk); for (pgoff_t i = 0; i < count; i++) { - u64 pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64); + u64 pte; int ret; + /* + * If not a leaf pt, skip unless non-leaf pt is interleaved between + * leaf ptes which causes the page walk to skip over the child leaves + */ + if (xe_child->base.children && xe_child->base.children[first + i]) { + u64 pt_size = 1ULL << walk->shifts[xe_child->level]; + bool edge_pt = (i == 0 && !IS_ALIGNED(addr, pt_size)) || + (i == count - 1 && !IS_ALIGNED(next, pt_size)); + + if (!edge_pt) { + xe_page_reclaim_list_abort(xe_walk->tile->primary_gt, + xe_walk->prl, + "PT is skipped by walk at level=%u offset=%lu", + xe_child->level, first + i); + break; + } + continue; + } + + pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64); + /* * In rare scenarios, pte may not be written yet due to racy conditions. * In such cases, invalidate the PRL and fallback to full PPC invalidation. @@ -1674,9 +1695,8 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, } /* Ensure it is a defined page */ - xe_tile_assert(xe_walk->tile, - xe_child->level == 0 || - (pte & (XE_PTE_PS64 | XE_PDE_PS_2M | XE_PDPE_PS_1G))); + xe_tile_assert(xe_walk->tile, xe_child->level == 0 || + (pte & (XE_PDE_PS_2M | XE_PDPE_PS_1G))); /* An entry should be added for 64KB but contigious 4K have XE_PTE_PS64 */ if (pte & XE_PTE_PS64) @@ -1701,11 +1721,11 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, killed = xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk); /* - * Verify PRL is active and if entry is not a leaf pte (base.children conditions), - * there is a potential need to invalidate the PRL if any PTE (num_live) are dropped. + * Verify if any PTE are potentially dropped at non-leaf levels, either from being + * killed or the page walk covers the region. */ - if (xe_walk->prl && level > 1 && xe_child->num_live && - xe_child->base.children && xe_child->base.children[first]) { + if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) && + xe_child->level > MAX_HUGEPTE_LEVEL && xe_child->num_live) { bool covered = xe_pt_covers(addr, next, xe_child->level, &xe_walk->base); /*