diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_irq.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_irq.h index e7183cf05776..383e89813db3 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_irq.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_irq.h @@ -14,6 +14,16 @@ void dpu_core_irq_uninstall(struct msm_kms *kms); irqreturn_t dpu_core_irq(struct msm_kms *kms); +#ifdef CONFIG_PREEMPT_RT +/** + * dpu_core_irq_thread - core IRQ threaded handler, dispatches the per-IRQ + * callbacks recorded by dpu_core_irq() + * @kms: MSM KMS handle + * @return: interrupt handling status + */ +irqreturn_t dpu_core_irq_thread(struct msm_kms *kms); +#endif + u32 dpu_core_irq_read( struct dpu_kms *dpu_kms, unsigned int irq_idx); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c index 5b7cd5241f45..95b016e8d7df 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c @@ -322,6 +322,52 @@ static void dpu_core_irq_callback_handler(struct dpu_kms *dpu_kms, unsigned int irq_entry->cb(irq_entry->arg); } +/* + * dpu_core_irq_dispatch() runs the fired bits for @reg_idx through their + * registered callbacks directly. Only used on non-PREEMPT_RT kernels, where + * dpu_core_irq() itself is allowed to take the sleepable locks reached via + * those callbacks. + */ +static void dpu_core_irq_dispatch(struct dpu_kms *dpu_kms, int reg_idx, u32 irq_status) +{ + unsigned int irq_idx; + int bit; + + /* + * Search through matching intr status. + */ + while ((bit = ffs(irq_status)) != 0) { + irq_idx = DPU_IRQ_IDX(reg_idx, bit - 1); + + dpu_core_irq_callback_handler(dpu_kms, irq_idx); + + /* + * When callback finish, clear the irq_status + * with the matching mask. Once irq_status + * is all cleared, the search can be stopped. + */ + irq_status &= ~BIT(bit - 1); + } +} + +#ifdef CONFIG_PREEMPT_RT +/* + * dpu_core_irq_defer_to_thread() hands the fired bits for @reg_idx off to + * dpu_core_irq_thread(), which dispatches them from a genuine preemptible + * IRQ thread instead of the hardirq context dpu_core_irq() runs in on + * PREEMPT_RT. + */ +static void dpu_core_irq_defer_to_thread(struct dpu_hw_intr *intr, int reg_idx, u32 irq_status) +{ + intr->irq_pending_mask[reg_idx] |= irq_status; +} +#else +static inline void dpu_core_irq_defer_to_thread(struct dpu_hw_intr *intr, int reg_idx, + u32 irq_status) +{ +} +#endif + /** * dpu_core_irq - core IRQ handler * @kms: MSM KMS handle @@ -332,16 +378,14 @@ irqreturn_t dpu_core_irq(struct msm_kms *kms) struct dpu_kms *dpu_kms = to_dpu_kms(kms); struct dpu_hw_intr *intr = dpu_kms->hw_intr; int reg_idx; - unsigned int irq_idx; u32 irq_status; u32 enable_mask; - int bit; - unsigned long irq_flags; + bool wake_thread = false; if (!intr) return IRQ_NONE; - spin_lock_irqsave(&intr->irq_lock, irq_flags); + raw_spin_lock(&intr->irq_lock); for (reg_idx = 0; reg_idx < MDP_INTR_MAX; reg_idx++) { if (!test_bit(reg_idx, &intr->irq_mask)) continue; @@ -363,6 +407,52 @@ irqreturn_t dpu_core_irq(struct msm_kms *kms) if (!irq_status) continue; + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + dpu_core_irq_defer_to_thread(intr, reg_idx, irq_status); + wake_thread = true; + } else { + dpu_core_irq_dispatch(dpu_kms, reg_idx, irq_status); + } + } + + /* ensure register writes go through */ + wmb(); + + raw_spin_unlock(&intr->irq_lock); + + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + return wake_thread ? IRQ_WAKE_THREAD : IRQ_NONE; + + return IRQ_HANDLED; +} + +#ifdef CONFIG_PREEMPT_RT +/* + * dpu_core_irq_thread() runs in a genuine preemptible IRQ thread (woken via + * IRQ_WAKE_THREAD from dpu_core_irq() above), so it's safe for it -- and the + * per-encoder callbacks it dispatches to -- to take spinlock_t/rt_mutex + * locks such as enc_spinlock, dpu_crtc::spin_lock, and the various DRM-core + * locks reached via vblank/CRC/writeback handling. + */ +irqreturn_t dpu_core_irq_thread(struct msm_kms *kms) +{ + struct dpu_kms *dpu_kms = to_dpu_kms(kms); + struct dpu_hw_intr *intr = dpu_kms->hw_intr; + int reg_idx; + unsigned int irq_idx; + u32 irq_status; + unsigned long irq_flags; + int bit; + + if (!intr) + return IRQ_NONE; + + for (reg_idx = 0; reg_idx < MDP_INTR_MAX; reg_idx++) { + raw_spin_lock_irqsave(&intr->irq_lock, irq_flags); + irq_status = intr->irq_pending_mask[reg_idx]; + intr->irq_pending_mask[reg_idx] = 0; + raw_spin_unlock_irqrestore(&intr->irq_lock, irq_flags); + /* * Search through matching intr status. */ @@ -380,13 +470,9 @@ irqreturn_t dpu_core_irq(struct msm_kms *kms) } } - /* ensure register writes go through */ - wmb(); - - spin_unlock_irqrestore(&intr->irq_lock, irq_flags); - return IRQ_HANDLED; } +#endif static int dpu_hw_intr_enable_irq_locked(struct dpu_hw_intr *intr, unsigned int irq_idx) @@ -410,7 +496,7 @@ static int dpu_hw_intr_enable_irq_locked(struct dpu_hw_intr *intr, * under irq_lock and it's the caller's responsibility to ensure that's * held. */ - assert_spin_locked(&intr->irq_lock); + assert_raw_spin_locked(&intr->irq_lock); reg_idx = DPU_IRQ_REG(irq_idx); reg = &intr->intr_set[reg_idx]; @@ -466,7 +552,7 @@ static int dpu_hw_intr_disable_irq_locked(struct dpu_hw_intr *intr, * under irq_lock and it's the caller's responsibility to ensure that's * held. */ - assert_spin_locked(&intr->irq_lock); + assert_raw_spin_locked(&intr->irq_lock); reg_idx = DPU_IRQ_REG(irq_idx); reg = &intr->intr_set[reg_idx]; @@ -554,7 +640,7 @@ u32 dpu_core_irq_read(struct dpu_kms *dpu_kms, return 0; } - spin_lock_irqsave(&intr->irq_lock, irq_flags); + raw_spin_lock_irqsave(&intr->irq_lock, irq_flags); reg_idx = DPU_IRQ_REG(irq_idx); intr_status = DPU_REG_READ(&intr->hw, @@ -567,7 +653,7 @@ u32 dpu_core_irq_read(struct dpu_kms *dpu_kms, /* ensure register writes go through */ wmb(); - spin_unlock_irqrestore(&intr->irq_lock, irq_flags); + raw_spin_unlock_irqrestore(&intr->irq_lock, irq_flags); return intr_status; } @@ -616,7 +702,7 @@ struct dpu_hw_intr *dpu_hw_intr_init(struct drm_device *dev, intr->irq_mask |= BIT(DPU_IRQ_REG(intf->intr_tear_rd_ptr)); } - spin_lock_init(&intr->irq_lock); + raw_spin_lock_init(&intr->irq_lock); return intr; } @@ -656,11 +742,11 @@ int dpu_core_irq_register_callback(struct dpu_kms *dpu_kms, VERB("[%pS] IRQ=[%d, %d]\n", __builtin_return_address(0), DPU_IRQ_REG(irq_idx), DPU_IRQ_BIT(irq_idx)); - spin_lock_irqsave(&dpu_kms->hw_intr->irq_lock, irq_flags); + raw_spin_lock_irqsave(&dpu_kms->hw_intr->irq_lock, irq_flags); irq_entry = dpu_core_irq_get_entry(dpu_kms->hw_intr, irq_idx); if (unlikely(WARN_ON(irq_entry->cb))) { - spin_unlock_irqrestore(&dpu_kms->hw_intr->irq_lock, irq_flags); + raw_spin_unlock_irqrestore(&dpu_kms->hw_intr->irq_lock, irq_flags); return -EBUSY; } @@ -675,7 +761,7 @@ int dpu_core_irq_register_callback(struct dpu_kms *dpu_kms, if (ret) DPU_ERROR("Failed/ to enable IRQ=[%d, %d]\n", DPU_IRQ_REG(irq_idx), DPU_IRQ_BIT(irq_idx)); - spin_unlock_irqrestore(&dpu_kms->hw_intr->irq_lock, irq_flags); + raw_spin_unlock_irqrestore(&dpu_kms->hw_intr->irq_lock, irq_flags); trace_dpu_irq_register_success(DPU_IRQ_REG(irq_idx), DPU_IRQ_BIT(irq_idx)); @@ -707,7 +793,7 @@ int dpu_core_irq_unregister_callback(struct dpu_kms *dpu_kms, VERB("[%pS] IRQ=[%d, %d]\n", __builtin_return_address(0), DPU_IRQ_REG(irq_idx), DPU_IRQ_BIT(irq_idx)); - spin_lock_irqsave(&dpu_kms->hw_intr->irq_lock, irq_flags); + raw_spin_lock_irqsave(&dpu_kms->hw_intr->irq_lock, irq_flags); trace_dpu_core_irq_unregister_callback(DPU_IRQ_REG(irq_idx), DPU_IRQ_BIT(irq_idx)); ret = dpu_hw_intr_disable_irq_locked(dpu_kms->hw_intr, irq_idx); @@ -719,7 +805,7 @@ int dpu_core_irq_unregister_callback(struct dpu_kms *dpu_kms, irq_entry->cb = NULL; irq_entry->arg = NULL; - spin_unlock_irqrestore(&dpu_kms->hw_intr->irq_lock, irq_flags); + raw_spin_unlock_irqrestore(&dpu_kms->hw_intr->irq_lock, irq_flags); trace_dpu_irq_unregister_success(DPU_IRQ_REG(irq_idx), DPU_IRQ_BIT(irq_idx)); @@ -736,11 +822,11 @@ static int dpu_debugfs_core_irq_show(struct seq_file *s, void *v) void *cb; for (i = 1; i <= DPU_NUM_IRQS; i++) { - spin_lock_irqsave(&dpu_kms->hw_intr->irq_lock, irq_flags); + raw_spin_lock_irqsave(&dpu_kms->hw_intr->irq_lock, irq_flags); irq_entry = dpu_core_irq_get_entry(dpu_kms->hw_intr, i); irq_count = atomic_read(&irq_entry->count); cb = irq_entry->cb; - spin_unlock_irqrestore(&dpu_kms->hw_intr->irq_lock, irq_flags); + raw_spin_unlock_irqrestore(&dpu_kms->hw_intr->irq_lock, irq_flags); if (irq_count || cb) seq_printf(s, "IRQ=[%d, %d] count:%d cb:%ps\n", diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h index 142358a105c5..2bd16f9341c2 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.h @@ -54,14 +54,21 @@ struct dpu_hw_intr_entry { * @ops: function pointer mapping for IRQ handling * @cache_irq_mask: array of IRQ enable masks reg storage created during init * @save_irq_status: array of IRQ status reg storage created during init - * @irq_lock: spinlock for accessing IRQ resources + * @irq_lock: raw spinlock for accessing IRQ resources. + * @irq_pending_mask: per-register bitmask of enabled+fired IRQs that the + * hardirq primary handler has acked in hardware but not + * yet handed off to the IRQ thread for callback dispatch + * (CONFIG_PREEMPT_RT only) * @irq_cb_tbl: array of IRQ callbacks */ struct dpu_hw_intr { struct dpu_hw_blk_reg_map hw; u32 cache_irq_mask[MDP_INTR_MAX]; u32 *save_irq_status; - spinlock_t irq_lock; + raw_spinlock_t irq_lock; +#ifdef CONFIG_PREEMPT_RT + u32 irq_pending_mask[MDP_INTR_MAX]; +#endif unsigned long irq_mask; const struct dpu_intr_reg *intr_set; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c index ac5dc844fead..72ebe2a9e720 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c @@ -1100,6 +1100,9 @@ static const struct msm_kms_funcs kms_funcs = { .irq_postinstall = dpu_irq_postinstall, .irq_uninstall = dpu_core_irq_uninstall, .irq = dpu_core_irq, +#ifdef CONFIG_PREEMPT_RT + .irq_thread = dpu_core_irq_thread, +#endif .enable_commit = dpu_kms_enable_commit, .disable_commit = dpu_kms_disable_commit, .check_mode_changed = dpu_kms_check_mode_changed, diff --git a/drivers/gpu/drm/msm/msm_kms.c b/drivers/gpu/drm/msm/msm_kms.c index 33ab30300e16..9868e7df381b 100644 --- a/drivers/gpu/drm/msm/msm_kms.c +++ b/drivers/gpu/drm/msm/msm_kms.c @@ -43,6 +43,20 @@ static irqreturn_t msm_irq(int irq, void *arg) return kms->funcs->irq(kms); } +#ifdef CONFIG_PREEMPT_RT +static irqreturn_t msm_irq_thread(int irq, void *arg) +{ + struct drm_device *dev = arg; + struct msm_drm_private *priv = dev->dev_private; + struct msm_kms *kms = priv->kms; + + if (WARN_ON_ONCE(!kms || !kms->funcs->irq_thread)) + return IRQ_NONE; + + return kms->funcs->irq_thread(kms); +} +#endif + static void msm_irq_preinstall(struct drm_device *dev) { struct msm_drm_private *priv = dev->dev_private; @@ -77,7 +91,25 @@ static int msm_irq_install(struct drm_device *dev, unsigned int irq) msm_irq_preinstall(dev); +#ifdef CONFIG_PREEMPT_RT + /* + * Some KMS backends (e.g. dpu1) split their handler into a minimal + * hardirq primary handler that only acks hardware and a threaded + * handler that does the actual (sleep-capable) callback dispatch. + * IRQF_ONESHOT keeps the primary handler running as a true hardirq + * even under PREEMPT_RT's forced-threading (see + * irq_setup_forced_threading() in kernel/irq/manage.c), the same + * property IRQF_NO_THREAD gives the backends that don't split their + * handler and must run their whole ->irq() in hardirq context. + */ + if (kms->funcs->irq_thread) + ret = request_threaded_irq(irq, msm_irq, msm_irq_thread, + IRQF_ONESHOT, dev->driver->name, dev); + else + ret = request_irq(irq, msm_irq, IRQF_NO_THREAD, dev->driver->name, dev); +#else ret = request_irq(irq, msm_irq, 0, dev->driver->name, dev); +#endif if (ret) return ret; diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h index f25b31e502d2..f0396cfbfc0c 100644 --- a/drivers/gpu/drm/msm/msm_kms.h +++ b/drivers/gpu/drm/msm/msm_kms.h @@ -30,6 +30,15 @@ struct msm_kms_funcs { int (*irq_postinstall)(struct msm_kms *kms); void (*irq_uninstall)(struct msm_kms *kms); irqreturn_t (*irq)(struct msm_kms *kms); +#ifdef CONFIG_PREEMPT_RT + /* + * Optional threaded companion to ->irq(), used only on PREEMPT_RT. + * When set, ->irq() must behave as a true hardirq handler (only + * raw_spinlock_t, no sleeping) and hand off any deferred work to + * ->irq_thread(), which runs in a real, preemptible IRQ thread. + */ + irqreturn_t (*irq_thread)(struct msm_kms *kms); +#endif int (*enable_vblank)(struct msm_kms *kms, struct drm_crtc *crtc); void (*disable_vblank)(struct msm_kms *kms, struct drm_crtc *crtc);