From 8848f0665b3cd4fbb3107b384f5205380c90634d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 15 Jun 2021 12:12:24 +0100 Subject: [PATCH 1/4] arm64: Add cpuidle context save/restore helpers As we need to start doing some additional work on all idle paths, let's introduce a set of macros that will perform the work related to the GICv3 pseudo-NMI idle entry exit. Stubs are introduced to 32bit ARM for compatibility. As these helpers are currently unused, there is no functional change. Tested-by: Valentin Schneider Reviewed-by: Lorenzo Pieralisi Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20210615111227.2454465-2-maz@kernel.org Signed-off-by: Will Deacon --- arch/arm/include/asm/cpuidle.h | 5 +++++ arch/arm64/include/asm/cpuidle.h | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/arch/arm/include/asm/cpuidle.h b/arch/arm/include/asm/cpuidle.h index 0d67ed682e07..dc8f53f1a219 100644 --- a/arch/arm/include/asm/cpuidle.h +++ b/arch/arm/include/asm/cpuidle.h @@ -49,4 +49,9 @@ extern int arm_cpuidle_suspend(int index); extern int arm_cpuidle_init(int cpu); +struct arm_cpuidle_irq_context { }; + +#define arm_cpuidle_save_irq_context(c) (void)c +#define arm_cpuidle_restore_irq_context(c) (void)c + #endif diff --git a/arch/arm64/include/asm/cpuidle.h b/arch/arm64/include/asm/cpuidle.h index 3c5ddb429ea2..14a19d1141bd 100644 --- a/arch/arm64/include/asm/cpuidle.h +++ b/arch/arm64/include/asm/cpuidle.h @@ -18,4 +18,39 @@ static inline int arm_cpuidle_suspend(int index) return -EOPNOTSUPP; } #endif + +#ifdef CONFIG_ARM64_PSEUDO_NMI +#include + +struct arm_cpuidle_irq_context { + unsigned long pmr; + unsigned long daif_bits; +}; + +#define arm_cpuidle_save_irq_context(__c) \ + do { \ + struct arm_cpuidle_irq_context *c = __c; \ + if (system_uses_irq_prio_masking()) { \ + c->daif_bits = read_sysreg(daif); \ + write_sysreg(c->daif_bits | PSR_I_BIT | PSR_F_BIT, \ + daif); \ + c->pmr = gic_read_pmr(); \ + gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); \ + } \ + } while (0) + +#define arm_cpuidle_restore_irq_context(__c) \ + do { \ + struct arm_cpuidle_irq_context *c = __c; \ + if (system_uses_irq_prio_masking()) { \ + gic_write_pmr(c->pmr); \ + write_sysreg(c->daif_bits, daif); \ + } \ + } while (0) +#else +struct arm_cpuidle_irq_context { }; + +#define arm_cpuidle_save_irq_context(c) (void)c +#define arm_cpuidle_restore_irq_context(c) (void)c +#endif #endif From d4dc10277255afc303de4f00cbee0b9ce74d870f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 15 Jun 2021 12:12:25 +0100 Subject: [PATCH 2/4] arm64: Convert cpu_do_idle() to using cpuidle context helpers Now that we have helpers that are aware of the pseudo-NMI feature, introduce them to cpu_do_idle(). This allows for some nice cleanup. No functional change intended. Tested-by: Valentin Schneider Reviewed-by: Lorenzo Pieralisi Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20210615111227.2454465-3-maz@kernel.org Signed-off-by: Will Deacon --- arch/arm64/kernel/process.c | 41 ++++++++----------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index b4bb67f17a2c..b715c6b2558f 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -46,9 +46,9 @@ #include #include -#include #include #include +#include #include #include #include @@ -74,33 +74,6 @@ EXPORT_SYMBOL_GPL(pm_power_off); void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); -static void noinstr __cpu_do_idle(void) -{ - dsb(sy); - wfi(); -} - -static void noinstr __cpu_do_idle_irqprio(void) -{ - unsigned long pmr; - unsigned long daif_bits; - - daif_bits = read_sysreg(daif); - write_sysreg(daif_bits | PSR_I_BIT | PSR_F_BIT, daif); - - /* - * Unmask PMR before going idle to make sure interrupts can - * be raised. - */ - pmr = gic_read_pmr(); - gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); - - __cpu_do_idle(); - - gic_write_pmr(pmr); - write_sysreg(daif_bits, daif); -} - /* * cpu_do_idle() * @@ -112,10 +85,14 @@ static void noinstr __cpu_do_idle_irqprio(void) */ void noinstr cpu_do_idle(void) { - if (system_uses_irq_prio_masking()) - __cpu_do_idle_irqprio(); - else - __cpu_do_idle(); + struct arm_cpuidle_irq_context context; + + arm_cpuidle_save_irq_context(&context); + + dsb(sy); + wfi(); + + arm_cpuidle_restore_irq_context(&context); } /* From c9223b616298c3d0e6ff5dd20d14d65c2131c535 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 15 Jun 2021 12:12:26 +0100 Subject: [PATCH 3/4] PSCI: Use cpuidle context helpers in psci_cpu_suspend_enter() The PSCI CPU suspend code isn't aware of the PMR vs DAIF game, resulting in a system that locks up if entering CPU suspend with GICv3 pNMI enabled. To save the day, teach the suspend code about our new cpuidle context helpers, which will do everything that's required just like the usual WFI cpuidle code. This fixes my Altra system, which would otherwise lock-up at boot time when booted with irqchip.gicv3_pseudo_nmi=1. Tested-by: Valentin Schneider Reviewed-by: Lorenzo Pieralisi Signed-off-by: Marc Zyngier Reviewed-by: Sudeep Holla Link: https://lore.kernel.org/r/20210615111227.2454465-4-maz@kernel.org Signed-off-by: Will Deacon --- drivers/firmware/psci/psci.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c index 3c1c5daf6df2..e3da38e15c5b 100644 --- a/drivers/firmware/psci/psci.c +++ b/drivers/firmware/psci/psci.c @@ -335,10 +335,15 @@ int psci_cpu_suspend_enter(u32 state) { int ret; - if (!psci_power_state_loses_context(state)) + if (!psci_power_state_loses_context(state)) { + struct arm_cpuidle_irq_context context; + + arm_cpuidle_save_irq_context(&context); ret = psci_ops.cpu_suspend(state, 0); - else + arm_cpuidle_restore_irq_context(&context); + } else { ret = cpu_suspend(state, psci_suspend_finisher); + } return ret; } From 77345ef70445a8f16e0685dade0d68bdf41f19d7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 15 Jun 2021 12:12:27 +0100 Subject: [PATCH 4/4] arm64: suspend: Use cpuidle context helpers in cpu_suspend() Use cpuidle context helpers to switch to using DAIF.IF instead of PMR to mask interrupts, ensuring that we suspend with interrupts being able to reach the CPU interface. Signed-off-by: Marc Zyngier Reviewed-by: Sudeep Holla Link: https://lore.kernel.org/r/20210615111227.2454465-5-maz@kernel.org Signed-off-by: Will Deacon --- arch/arm64/kernel/suspend.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index e3f72df9509d..938ce6fbee8a 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -91,6 +92,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) int ret = 0; unsigned long flags; struct sleep_stack_data state; + struct arm_cpuidle_irq_context context; /* Report any MTE async fault before going to suspend */ mte_suspend_enter(); @@ -103,12 +105,18 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) flags = local_daif_save(); /* - * Function graph tracer state gets incosistent when the kernel + * Function graph tracer state gets inconsistent when the kernel * calls functions that never return (aka suspend finishers) hence * disable graph tracing during their execution. */ pause_graph_tracing(); + /* + * Switch to using DAIF.IF instead of PMR in order to reliably + * resume if we're using pseudo-NMIs. + */ + arm_cpuidle_save_irq_context(&context); + if (__cpu_suspend_enter(&state)) { /* Call the suspend finisher */ ret = fn(arg); @@ -126,6 +134,8 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) RCU_NONIDLE(__cpu_suspend_exit()); } + arm_cpuidle_restore_irq_context(&context); + unpause_graph_tracing(); /*