From 2d0d656700d67239a57afaf617439143d8dac9be Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:39 +0100 Subject: [PATCH 01/39] arm64: Add Neoverse-N2, Cortex-A710 CPU part definition Add the CPU Partnumbers for the new Arm designs. Cc: Catalin Marinas Cc: Mark Rutland Cc: Will Deacon Acked-by: Catalin Marinas Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-2-suzuki.poulose@arm.com Signed-off-by: Will Deacon --- arch/arm64/include/asm/cputype.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 6231e1f0abe7..19b8441aa8f2 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -73,6 +73,8 @@ #define ARM_CPU_PART_CORTEX_A76 0xD0B #define ARM_CPU_PART_NEOVERSE_N1 0xD0C #define ARM_CPU_PART_CORTEX_A77 0xD0D +#define ARM_CPU_PART_CORTEX_A710 0xD47 +#define ARM_CPU_PART_NEOVERSE_N2 0xD49 #define APM_CPU_PART_POTENZA 0x000 @@ -113,6 +115,8 @@ #define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76) #define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1) #define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77) +#define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710) +#define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) From b9d216fcef4298de76519e2baeed69ba482467bd Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:40 +0100 Subject: [PATCH 02/39] arm64: errata: Add detection for TRBE overwrite in FILL mode Arm Neoverse-N2 and the Cortex-A710 cores are affected by a CPU erratum where the TRBE will overwrite the trace buffer in FILL mode. The TRBE doesn't stop (as expected in FILL mode) when it reaches the limit and wraps to the base to continue writing upto 3 cache lines. This will overwrite any trace that was written previously. Add the Neoverse-N2 erratum(#2139208) and Cortex-A710 erratum (#2119858) to the detection logic. This will be used by the TRBE driver in later patches to work around the issue. The detection has been kept with the core arm64 errata framework list to make sure : - We don't duplicate the framework in TRBE driver - The errata detection is advertised like the rest of the CPU errata. Note that the Kconfig entries are not fully active until the TRBE driver implements the work around. Cc: Will Deacon Cc: Mark Rutland Cc: Anshuman Khandual Cc: Catalin Marinas Cc: Mathieu Poirier Cc: Mike Leach cc: Leo Yan Acked-by: Catalin Marinas Reviewed-by: Mathieu Poirier Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-3-suzuki.poulose@arm.com Signed-off-by: Will Deacon --- Documentation/arm64/silicon-errata.rst | 4 +++ arch/arm64/Kconfig | 41 ++++++++++++++++++++++++++ arch/arm64/kernel/cpu_errata.c | 25 ++++++++++++++++ arch/arm64/tools/cpucaps | 1 + 4 files changed, 71 insertions(+) diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst index d410a47ffa57..2f99229d993c 100644 --- a/Documentation/arm64/silicon-errata.rst +++ b/Documentation/arm64/silicon-errata.rst @@ -92,12 +92,16 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1508412 | ARM64_ERRATUM_1508412 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1349291 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1542419 | ARM64_ERRATUM_1542419 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N2 | #2139208 | ARM64_ERRATUM_2139208 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #841119,826419 | N/A | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 5c7ae4c3954b..26bd128935bc 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -666,6 +666,47 @@ config ARM64_ERRATUM_1508412 If unsure, say Y. +config ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE + bool + +config ARM64_ERRATUM_2119858 + bool "Cortex-A710: 2119858: workaround TRBE overwriting trace data in FILL mode" + default y + depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in + depends on CORESIGHT_TRBE + select ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE + help + This option adds the workaround for ARM Cortex-A710 erratum 2119858. + + Affected Cortex-A710 cores could overwrite up to 3 cache lines of trace + data at the base of the buffer (pointed to by TRBASER_EL1) in FILL mode in + the event of a WRAP event. + + Work around the issue by always making sure we move the TRBPTR_EL1 by + 256 bytes before enabling the buffer and filling the first 256 bytes of + the buffer with ETM ignore packets upon disabling. + + If unsure, say Y. + +config ARM64_ERRATUM_2139208 + bool "Neoverse-N2: 2139208: workaround TRBE overwriting trace data in FILL mode" + default y + depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in + depends on CORESIGHT_TRBE + select ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE + help + This option adds the workaround for ARM Neoverse-N2 erratum 2139208. + + Affected Neoverse-N2 cores could overwrite up to 3 cache lines of trace + data at the base of the buffer (pointed to by TRBASER_EL1) in FILL mode in + the event of a WRAP event. + + Work around the issue by always making sure we move the TRBPTR_EL1 by + 256 bytes before enabling the buffer and filling the first 256 bytes of + the buffer with ETM ignore packets upon disabling. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index e2c20c036442..ccd757373f36 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -340,6 +340,18 @@ static const struct midr_range erratum_1463225[] = { }; #endif +#ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE +static const struct midr_range trbe_overwrite_fill_mode_cpus[] = { +#ifdef CONFIG_ARM64_ERRATUM_2139208 + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), +#endif +#ifdef CONFIG_ARM64_ERRATUM_2119858 + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), +#endif + {}, +}; +#endif /* CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE */ + const struct arm64_cpu_capabilities arm64_errata[] = { #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { @@ -533,6 +545,19 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .capability = ARM64_WORKAROUND_NVIDIA_CARMEL_CNP, ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), }, +#endif +#ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE + { + /* + * The erratum work around is handled within the TRBE + * driver and can be applied per-cpu. So, we can allow + * a late CPU to come online with this erratum. + */ + .desc = "ARM erratum 2119858 or 2139208", + .capability = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE, + .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, + CAP_MIDR_RANGE_LIST(trbe_overwrite_fill_mode_cpus), + }, #endif { } diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 49305c2e6dfd..1ccb92165bd8 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -53,6 +53,7 @@ WORKAROUND_1418040 WORKAROUND_1463225 WORKAROUND_1508412 WORKAROUND_1542419 +WORKAROUND_TRBE_OVERWRITE_FILL_MODE WORKAROUND_CAVIUM_23154 WORKAROUND_CAVIUM_27456 WORKAROUND_CAVIUM_30115 From fa82d0b4b833790ac4572377fb777dcea24a9d69 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:41 +0100 Subject: [PATCH 03/39] arm64: errata: Add workaround for TSB flush failures Arm Neoverse-N2 (#2067961) and Cortex-A710 (#2054223) suffers from errata, where a TSB (trace synchronization barrier) fails to flush the trace data completely, when executed from a trace prohibited region. In Linux we always execute it after we have moved the PE to trace prohibited region. So, we can apply the workaround every time a TSB is executed. The work around is to issue two TSB consecutively. NOTE: This errata is defined as LOCAL_CPU_ERRATUM, implying that a late CPU could be blocked from booting if it is the first CPU that requires the workaround. This is because we do not allow setting a cpu_hwcaps after the SMP boot. The other alternative is to use "this_cpu_has_cap()" instead of the faster system wide check, which may be a bit of an overhead, given we may have to do this in nvhe KVM host before a guest entry. Cc: Will Deacon Cc: Catalin Marinas Cc: Mathieu Poirier Cc: Mike Leach Cc: Mark Rutland Cc: Anshuman Khandual Cc: Marc Zyngier Acked-by: Catalin Marinas Reviewed-by: Mathieu Poirier Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-4-suzuki.poulose@arm.com Signed-off-by: Will Deacon --- Documentation/arm64/silicon-errata.rst | 4 ++++ arch/arm64/Kconfig | 33 ++++++++++++++++++++++++++ arch/arm64/include/asm/barrier.h | 16 ++++++++++++- arch/arm64/kernel/cpu_errata.c | 19 +++++++++++++++ arch/arm64/tools/cpucaps | 1 + 5 files changed, 72 insertions(+), 1 deletion(-) diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst index 2f99229d993c..569a92411dcd 100644 --- a/Documentation/arm64/silicon-errata.rst +++ b/Documentation/arm64/silicon-errata.rst @@ -94,6 +94,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A710 | #2054223 | ARM64_ERRATUM_2054223 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1349291 | N/A | @@ -102,6 +104,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2139208 | ARM64_ERRATUM_2139208 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N2 | #2067961 | ARM64_ERRATUM_2067961 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #841119,826419 | N/A | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 26bd128935bc..ec3bb346957f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -707,6 +707,39 @@ config ARM64_ERRATUM_2139208 If unsure, say Y. +config ARM64_WORKAROUND_TSB_FLUSH_FAILURE + bool + +config ARM64_ERRATUM_2054223 + bool "Cortex-A710: 2054223: workaround TSB instruction failing to flush trace" + default y + select ARM64_WORKAROUND_TSB_FLUSH_FAILURE + help + Enable workaround for ARM Cortex-A710 erratum 2054223 + + Affected cores may fail to flush the trace data on a TSB instruction, when + the PE is in trace prohibited state. This will cause losing a few bytes + of the trace cached. + + Workaround is to issue two TSB consecutively on affected cores. + + If unsure, say Y. + +config ARM64_ERRATUM_2067961 + bool "Neoverse-N2: 2067961: workaround TSB instruction failing to flush trace" + default y + select ARM64_WORKAROUND_TSB_FLUSH_FAILURE + help + Enable workaround for ARM Neoverse-N2 erratum 2067961 + + Affected cores may fail to flush the trace data on a TSB instruction, when + the PE is in trace prohibited state. This will cause losing a few bytes + of the trace cached. + + Workaround is to issue two TSB consecutively on affected cores. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 451e11e5fd23..1c5a00598458 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -23,7 +23,7 @@ #define dsb(opt) asm volatile("dsb " #opt : : : "memory") #define psb_csync() asm volatile("hint #17" : : : "memory") -#define tsb_csync() asm volatile("hint #18" : : : "memory") +#define __tsb_csync() asm volatile("hint #18" : : : "memory") #define csdb() asm volatile("hint #20" : : : "memory") #ifdef CONFIG_ARM64_PSEUDO_NMI @@ -46,6 +46,20 @@ #define dma_rmb() dmb(oshld) #define dma_wmb() dmb(oshst) + +#define tsb_csync() \ + do { \ + /* \ + * CPUs affected by Arm Erratum 2054223 or 2067961 needs \ + * another TSB to ensure the trace is flushed. The barriers \ + * don't have to be strictly back to back, as long as the \ + * CPU is in trace prohibited state. \ + */ \ + if (cpus_have_final_cap(ARM64_WORKAROUND_TSB_FLUSH_FAILURE)) \ + __tsb_csync(); \ + __tsb_csync(); \ + } while (0) + /* * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz * and 0 otherwise. diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index ccd757373f36..aaa66c9eee24 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -352,6 +352,18 @@ static const struct midr_range trbe_overwrite_fill_mode_cpus[] = { }; #endif /* CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE */ +#ifdef CONFIG_ARM64_WORKAROUND_TSB_FLUSH_FAILURE +static const struct midr_range tsb_flush_fail_cpus[] = { +#ifdef CONFIG_ARM64_ERRATUM_2067961 + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), +#endif +#ifdef CONFIG_ARM64_ERRATUM_2054223 + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), +#endif + {}, +}; +#endif /* CONFIG_ARM64_WORKAROUND_TSB_FLUSH_FAILURE */ + const struct arm64_cpu_capabilities arm64_errata[] = { #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { @@ -558,6 +570,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, CAP_MIDR_RANGE_LIST(trbe_overwrite_fill_mode_cpus), }, +#endif +#ifdef CONFIG_ARM64_WORKAROUND_TSB_FLUSH_FAILURE + { + .desc = "ARM erratum 2067961 or 2054223", + .capability = ARM64_WORKAROUND_TSB_FLUSH_FAILURE, + ERRATA_MIDR_RANGE_LIST(tsb_flush_fail_cpus), + }, #endif { } diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 1ccb92165bd8..2102e15af43d 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -54,6 +54,7 @@ WORKAROUND_1463225 WORKAROUND_1508412 WORKAROUND_1542419 WORKAROUND_TRBE_OVERWRITE_FILL_MODE +WORKAROUND_TSB_FLUSH_FAILURE WORKAROUND_CAVIUM_23154 WORKAROUND_CAVIUM_27456 WORKAROUND_CAVIUM_30115 From 8d81b2a38ddfc4b03662d2359765648c8b4cc73c Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:42 +0100 Subject: [PATCH 04/39] arm64: errata: Add detection for TRBE write to out-of-range Arm Neoverse-N2 and Cortex-A710 cores are affected by an erratum where the trbe, under some circumstances, might write upto 64bytes to an address after the Limit as programmed by the TRBLIMITR_EL1.LIMIT. This might - - Corrupt a page in the ring buffer, which may corrupt trace from a previous session, consumed by userspace. - Hit the guard page at the end of the vmalloc area and raise a fault. To keep the handling simpler, we always leave the last page from the range, which TRBE is allowed to write. This can be achieved by ensuring that we always have more than a PAGE worth space in the range, while calculating the LIMIT for TRBE. And then the LIMIT pointer can be adjusted to leave the PAGE (TRBLIMITR.LIMIT -= PAGE_SIZE), out of the TRBE range while enabling it. This makes sure that the TRBE will only write to an area within its allowed limit (i.e, [head-head+size]) and we do not have to handle address faults within the driver. Cc: Anshuman Khandual Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Cc: Will Deacon Cc: Mark Rutland Reviewed-by: Anshuman Khandual Reviewed-by: Mathieu Poirier Acked-by: Catalin Marinas Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-5-suzuki.poulose@arm.com Signed-off-by: Will Deacon --- Documentation/arm64/silicon-errata.rst | 4 +++ arch/arm64/Kconfig | 41 ++++++++++++++++++++++++++ arch/arm64/kernel/cpu_errata.c | 20 +++++++++++++ arch/arm64/tools/cpucaps | 1 + 4 files changed, 66 insertions(+) diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst index 569a92411dcd..5342e895fb60 100644 --- a/Documentation/arm64/silicon-errata.rst +++ b/Documentation/arm64/silicon-errata.rst @@ -96,6 +96,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2054223 | ARM64_ERRATUM_2054223 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A710 | #2224489 | ARM64_ERRATUM_2224489 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1349291 | N/A | @@ -106,6 +108,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2067961 | ARM64_ERRATUM_2067961 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N2 | #2253138 | ARM64_ERRATUM_2253138 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #841119,826419 | N/A | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index ec3bb346957f..a500af15ebd5 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -740,6 +740,47 @@ config ARM64_ERRATUM_2067961 If unsure, say Y. +config ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE + bool + +config ARM64_ERRATUM_2253138 + bool "Neoverse-N2: 2253138: workaround TRBE writing to address out-of-range" + depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in + depends on CORESIGHT_TRBE + default y + select ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE + help + This option adds the workaround for ARM Neoverse-N2 erratum 2253138. + + Affected Neoverse-N2 cores might write to an out-of-range address, not reserved + for TRBE. Under some conditions, the TRBE might generate a write to the next + virtually addressed page following the last page of the TRBE address space + (i.e., the TRBLIMITR_EL1.LIMIT), instead of wrapping around to the base. + + Work around this in the driver by always making sure that there is a + page beyond the TRBLIMITR_EL1.LIMIT, within the space allowed for the TRBE. + + If unsure, say Y. + +config ARM64_ERRATUM_2224489 + bool "Cortex-A710: 2224489: workaround TRBE writing to address out-of-range" + depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in + depends on CORESIGHT_TRBE + default y + select ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE + help + This option adds the workaround for ARM Cortex-A710 erratum 2224489. + + Affected Cortex-A710 cores might write to an out-of-range address, not reserved + for TRBE. Under some conditions, the TRBE might generate a write to the next + virtually addressed page following the last page of the TRBE address space + (i.e., the TRBLIMITR_EL1.LIMIT), instead of wrapping around to the base. + + Work around this in the driver by always making sure that there is a + page beyond the TRBLIMITR_EL1.LIMIT, within the space allowed for the TRBE. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index aaa66c9eee24..9e1c1aef9ebd 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -364,6 +364,18 @@ static const struct midr_range tsb_flush_fail_cpus[] = { }; #endif /* CONFIG_ARM64_WORKAROUND_TSB_FLUSH_FAILURE */ +#ifdef CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE +static struct midr_range trbe_write_out_of_range_cpus[] = { +#ifdef CONFIG_ARM64_ERRATUM_2253138 + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), +#endif +#ifdef CONFIG_ARM64_ERRATUM_2224489 + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), +#endif + {}, +}; +#endif /* CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE */ + const struct arm64_cpu_capabilities arm64_errata[] = { #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { @@ -577,6 +589,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .capability = ARM64_WORKAROUND_TSB_FLUSH_FAILURE, ERRATA_MIDR_RANGE_LIST(tsb_flush_fail_cpus), }, +#endif +#ifdef CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE + { + .desc = "ARM erratum 2253138 or 2224489", + .capability = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE, + .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, + CAP_MIDR_RANGE_LIST(trbe_write_out_of_range_cpus), + }, #endif { } diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 2102e15af43d..90628638e0f9 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -55,6 +55,7 @@ WORKAROUND_1508412 WORKAROUND_1542419 WORKAROUND_TRBE_OVERWRITE_FILL_MODE WORKAROUND_TSB_FLUSH_FAILURE +WORKAROUND_TRBE_WRITE_OUT_OF_RANGE WORKAROUND_CAVIUM_23154 WORKAROUND_CAVIUM_27456 WORKAROUND_CAVIUM_30115 From 692c9a499b286ea478f41b23a91fe3873b9e1326 Mon Sep 17 00:00:00 2001 From: Tao Zhang Date: Thu, 19 Aug 2021 17:29:37 +0800 Subject: [PATCH 05/39] coresight: cti: Correct the parameter for pm_runtime_put The input parameter of the function pm_runtime_put should be the same in the function cti_enable_hw and cti_disable_hw. The correct parameter to use here should be dev->parent. Signed-off-by: Tao Zhang Reviewed-by: Leo Yan Fixes: 835d722ba10a ("coresight: cti: Initial CoreSight CTI Driver") Cc: stable Link: https://lore.kernel.org/r/1629365377-5937-1-git-send-email-quic_taozha@quicinc.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-cti-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index e2a3620cbf48..8988b2ed2ea6 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -175,7 +175,7 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); spin_unlock(&drvdata->spinlock); - pm_runtime_put(dev); + pm_runtime_put(dev->parent); return 0; /* not disabled this call */ From 204879e6990d2a57d7a6e26792cec34f97a63c0e Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Fri, 3 Sep 2021 18:28:54 -0700 Subject: [PATCH 06/39] coresight: cpu-debug: Control default behavior via Kconfig Debugfs is nice and so are module parameters, but * debugfs doesn't take effect early (e.g., if drivers are locking up before user space gets anywhere) * module parameters either add a lot to the kernel command line, or else take effect late as well (if you build =m and configure in /etc/modprobe.d/) So in the same spirit as these CONFIG_PANIC_ON_OOPS (also available via cmdline or modparam) CONFIG_INTEL_IOMMU_DEFAULT_ON (also available via cmdline) add a new Kconfig option. Module parameters and debugfs can still override. Signed-off-by: Brian Norris Reviewed-by: Leo Yan [Fixed missing double quote in Kconfig title] Link: https://lore.kernel.org/r/20210903182839.1.I20856983f2841b78936134dcf9cdf6ecafe632b9@changeid Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/Kconfig | 13 +++++++++++++ drivers/hwtracing/coresight/coresight-cpu-debug.c | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/Kconfig b/drivers/hwtracing/coresight/Kconfig index f026e5c0e777..514a9b8086e3 100644 --- a/drivers/hwtracing/coresight/Kconfig +++ b/drivers/hwtracing/coresight/Kconfig @@ -150,6 +150,19 @@ config CORESIGHT_CPU_DEBUG To compile this driver as a module, choose M here: the module will be called coresight-cpu-debug. +config CORESIGHT_CPU_DEBUG_DEFAULT_ON + bool "Enable CoreSight CPU Debug by default" + depends on CORESIGHT_CPU_DEBUG + help + Say Y here to enable the CoreSight Debug panic-debug by default. This + can also be enabled via debugfs, but this ensures the debug feature + is enabled as early as possible. + + Has the same effect as setting coresight_cpu_debug.enable=1 on the + kernel command line. + + Say N if unsure. + config CORESIGHT_CTI tristate "CoreSight Cross Trigger Interface (CTI) driver" depends on ARM || ARM64 diff --git a/drivers/hwtracing/coresight/coresight-cpu-debug.c b/drivers/hwtracing/coresight/coresight-cpu-debug.c index 00de46565bc4..8845ec4b4402 100644 --- a/drivers/hwtracing/coresight/coresight-cpu-debug.c +++ b/drivers/hwtracing/coresight/coresight-cpu-debug.c @@ -105,7 +105,7 @@ static DEFINE_PER_CPU(struct debug_drvdata *, debug_drvdata); static int debug_count; static struct dentry *debug_debugfs_dir; -static bool debug_enable; +static bool debug_enable = IS_ENABLED(CONFIG_CORESIGHT_CPU_DEBUG_DEFAULT_ON); module_param_named(enable, debug_enable, bool, 0600); MODULE_PARM_DESC(enable, "Control to enable coresight CPU debug functionality"); From 0ab47f8079f27edc44ea0bcc67078561bcfdf542 Mon Sep 17 00:00:00 2001 From: Tanmay Jagdale Date: Wed, 1 Sep 2021 18:40:48 +0530 Subject: [PATCH 07/39] dt-bindings: coresight: Add burst size for TMC Add "arm,max-burst-size" optional property for TMC ETR. If specified, this value indicates the maximum burst size that can be initiated by TMC on the AXI bus. Signed-off-by: Tanmay Jagdale Reviewed-by: Mike Leach Acked-by: Rob Herring Link: https://lore.kernel.org/r/20210901131049.1365367-2-tanmay@marvell.com Signed-off-by: Mathieu Poirier --- Documentation/devicetree/bindings/arm/coresight.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/devicetree/bindings/arm/coresight.txt b/Documentation/devicetree/bindings/arm/coresight.txt index 7f9c1ca87487..c68d93a35b6c 100644 --- a/Documentation/devicetree/bindings/arm/coresight.txt +++ b/Documentation/devicetree/bindings/arm/coresight.txt @@ -127,6 +127,11 @@ its hardware characteristcs. * arm,scatter-gather: boolean. Indicates that the TMC-ETR can safely use the SG mode on this system. + * arm,max-burst-size: The maximum burst size initiated by TMC on the + AXI master interface. The burst size can be in the range [0..15], + the setting supports one data transfer per burst up to a maximum of + 16 data transfers per burst. + * Optional property for CATU : * interrupts : Exactly one SPI may be listed for reporting the address error From 4d5d88baa6c838bf92ed6a63c50dd3167c5a4956 Mon Sep 17 00:00:00 2001 From: Tanmay Jagdale Date: Wed, 1 Sep 2021 18:40:49 +0530 Subject: [PATCH 08/39] coresight: tmc: Configure AXI write burst size The current driver sets the write burst size initiated by TMC-ETR on AXI bus to a fixed value of 16. Make this configurable by reading the value specified in fwnode. If not specified, then default to 16. Introduced a "max_burst_size" variable in tmc_drvdata structure to facilitate this change. Signed-off-by: Tanmay Jagdale Reviewed-by: Mike Leach Link: https://lore.kernel.org/r/20210901131049.1365367-3-tanmay@marvell.com Signed-off-by: Mathieu Poirier --- .../hwtracing/coresight/coresight-tmc-core.c | 21 +++++++++++++++++-- .../hwtracing/coresight/coresight-tmc-etr.c | 3 ++- drivers/hwtracing/coresight/coresight-tmc.h | 6 +++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index 74c6323d4d6a..d0276af82494 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -432,6 +432,21 @@ static u32 tmc_etr_get_default_buffer_size(struct device *dev) return size; } +static u32 tmc_etr_get_max_burst_size(struct device *dev) +{ + u32 burst_size; + + if (fwnode_property_read_u32(dev->fwnode, "arm,max-burst-size", + &burst_size)) + return TMC_AXICTL_WR_BURST_16; + + /* Only permissible values are 0 to 15 */ + if (burst_size > 0xF) + burst_size = TMC_AXICTL_WR_BURST_16; + + return burst_size; +} + static int tmc_probe(struct amba_device *adev, const struct amba_id *id) { int ret = 0; @@ -469,10 +484,12 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id) /* This device is not associated with a session */ drvdata->pid = -1; - if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) + if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) { drvdata->size = tmc_etr_get_default_buffer_size(dev); - else + drvdata->max_burst_size = tmc_etr_get_max_burst_size(dev); + } else { drvdata->size = readl_relaxed(drvdata->base + TMC_RSZ) * 4; + } desc.dev = dev; desc.groups = coresight_tmc_groups; diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index acdb59e0e661..0ac2a611110b 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -982,7 +982,8 @@ static void __tmc_etr_enable_hw(struct tmc_drvdata *drvdata) axictl = readl_relaxed(drvdata->base + TMC_AXICTL); axictl &= ~TMC_AXICTL_CLEAR_MASK; - axictl |= (TMC_AXICTL_PROT_CTL_B1 | TMC_AXICTL_WR_BURST_16); + axictl |= TMC_AXICTL_PROT_CTL_B1; + axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size); axictl |= TMC_AXICTL_AXCACHE_OS; if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) { diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h index b91ec7dde7bc..6bec20a392b3 100644 --- a/drivers/hwtracing/coresight/coresight-tmc.h +++ b/drivers/hwtracing/coresight/coresight-tmc.h @@ -70,7 +70,8 @@ #define TMC_AXICTL_PROT_CTL_B0 BIT(0) #define TMC_AXICTL_PROT_CTL_B1 BIT(1) #define TMC_AXICTL_SCT_GAT_MODE BIT(7) -#define TMC_AXICTL_WR_BURST_16 0xF00 +#define TMC_AXICTL_WR_BURST(v) (((v) & 0xf) << 8) +#define TMC_AXICTL_WR_BURST_16 0xf /* Write-back Read and Write-allocate */ #define TMC_AXICTL_AXCACHE_OS (0xf << 2) #define TMC_AXICTL_ARCACHE_OS (0xf << 16) @@ -174,6 +175,8 @@ struct etr_buf { * @etr_buf: details of buffer used in TMC-ETR * @len: size of the available trace for ETF/ETB. * @size: trace buffer size for this TMC (common for all modes). + * @max_burst_size: The maximum burst size that can be initiated by + * TMC-ETR on AXI bus. * @mode: how this TMC is being used. * @config_type: TMC variant, must be of type @tmc_config_type. * @memwidth: width of the memory interface databus, in bytes. @@ -198,6 +201,7 @@ struct tmc_drvdata { }; u32 len; u32 size; + u32 max_burst_size; u32 mode; enum tmc_config_type config_type; enum tmc_mem_intf_width memwidth; From 26701ceb4c2c6bfa5f8c8984fa1a1ea08fa0f02c Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Mon, 9 Aug 2021 19:14:00 +0800 Subject: [PATCH 09/39] coresight: tmc-etr: Add barrier after updating AUX ring buffer Since a memory barrier is required between AUX trace data store and aux_head store, and the AUX trace data is filled with memcpy(), it's sufficient to use smp_wmb() so can ensure the trace data is visible prior to updating aux_head. Signed-off-by: Leo Yan Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210809111407.596077-3-leo.yan@linaro.org Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-tmc-etr.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 0ac2a611110b..7f4654b72314 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1564,6 +1564,14 @@ tmc_update_etr_buffer(struct coresight_device *csdev, */ if (etr_perf->snapshot) handle->head += size; + + /* + * Ensure that the AUX trace data is visible before the aux_head + * is updated via perf_aux_output_end(), as expected by the + * perf ring buffer. + */ + smp_wmb(); + out: /* * Don't set the TRUNCATED flag in snapshot mode because 1) the From bd8d06886d0a9b59d020fdb2496c76db77816768 Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Mon, 9 Aug 2021 19:14:01 +0800 Subject: [PATCH 10/39] coresight: tmc-etf: Add comment for store ordering Since the function CS_LOCK() has contained memory barrier mb(), it ensures the visibility of the AUX trace data before updating the aux_head, thus it's needless to add any explicit barrier anymore. Add comment to make clear for the barrier usage for ETF. Signed-off-by: Leo Yan Link: https://lore.kernel.org/r/20210809111407.596077-4-leo.yan@linaro.org Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-tmc-etf.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index cd0fb7bfba68..8debd4f40f06 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -553,6 +553,11 @@ static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev, if (buf->snapshot) handle->head += to_read; + /* + * CS_LOCK() contains mb() so it can ensure visibility of the AUX trace + * data before the aux_head is updated via perf_aux_output_end(), which + * is expected by the perf ring buffer. + */ CS_LOCK(drvdata->base); out: spin_unlock_irqrestore(&drvdata->spinlock, flags); From f36dec8da1a4a083c51e4d2297985d3648b6740f Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Sun, 12 Sep 2021 20:57:47 +0800 Subject: [PATCH 11/39] coresight: tmc-etr: Use perf_output_handle::head for AUX ring buffer When enable the Arm CoreSight PMU event, the context for AUX ring buffer is prepared in the structure perf_output_handle, and its field "head" points the head of the AUX ring buffer and it is updated after filling AUX trace data into buffer. Current code uses an extra field etr_perf_buffer::head to maintain the header for the AUX ring buffer which is not necessary; alternatively, it's better to directly use perf_output_handle::head. This patch removes the field etr_perf_buffer::head and directly uses perf_output_handle::head for the head of AUX ring buffer. Signed-off-by: Leo Yan Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210912125748.2816606-2-leo.yan@linaro.org Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-tmc-etr.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 7f4654b72314..14a86a232a18 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -32,7 +32,6 @@ struct etr_flat_buf { * @etr_buf - Actual buffer used by the ETR * @pid - The PID this etr_perf_buffer belongs to. * @snaphost - Perf session mode - * @head - handle->head at the beginning of the session. * @nr_pages - Number of pages in the ring buffer. * @pages - Array of Pages in the ring buffer. */ @@ -41,7 +40,6 @@ struct etr_perf_buffer { struct etr_buf *etr_buf; pid_t pid; bool snapshot; - unsigned long head; int nr_pages; void **pages; }; @@ -1438,16 +1436,16 @@ free_etr_perf_buffer: * buffer to the perf ring buffer. */ static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf, + unsigned long head, unsigned long src_offset, unsigned long to_copy) { long bytes; long pg_idx, pg_offset; - unsigned long head = etr_perf->head; char **dst_pages, *src_buf; struct etr_buf *etr_buf = etr_perf->etr_buf; - head = etr_perf->head; + head = PERF_IDX2OFF(head, etr_perf); pg_idx = head >> PAGE_SHIFT; pg_offset = head & (PAGE_SIZE - 1); dst_pages = (char **)etr_perf->pages; @@ -1554,7 +1552,7 @@ tmc_update_etr_buffer(struct coresight_device *csdev, /* Insert barrier packets at the beginning, if there was an overflow */ if (lost) tmc_etr_buf_insert_barrier_packet(etr_buf, offset); - tmc_etr_sync_perf_buffer(etr_perf, offset, size); + tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size); /* * In snapshot mode we simply increment the head by the number of byte @@ -1614,8 +1612,6 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data) goto unlock_out; } - etr_perf->head = PERF_IDX2OFF(handle->head, etr_perf); - /* * No HW configuration is needed if the sink is already in * use for this session. From 7ba7ae1d5a4737ebea582526ada9d79c74dfd75d Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Sun, 12 Sep 2021 20:57:48 +0800 Subject: [PATCH 12/39] coresight: Update comments for removing cs_etm_find_snapshot() Commit 2f01c200d440 ("perf cs-etm: Remove callback cs_etm_find_snapshot()") has removed the function cs_etm_find_snapshot() from the perf tool in the user space, now CoreSight trace directly uses the perf common function __auxtrace_mmap__read() to calcualte the head and size for AUX trace data in snapshot mode. This patch updates the comments in drivers to make them generic and not stick to any specific function from perf tool. Signed-off-by: Leo Yan Link: https://lore.kernel.org/r/20210912125748.2816606-3-leo.yan@linaro.org Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-etb10.c | 5 ++--- drivers/hwtracing/coresight/coresight-tmc-etf.c | 5 ++--- drivers/hwtracing/coresight/coresight-tmc-etr.c | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index f775cbee12b8..efa39820acec 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -557,9 +557,8 @@ static unsigned long etb_update_buffer(struct coresight_device *csdev, /* * In snapshot mode we simply increment the head by the number of byte - * that were written. User space function cs_etm_find_snapshot() will - * figure out how many bytes to get from the AUX buffer based on the - * position of the head. + * that were written. User space will figure out how many bytes to get + * from the AUX buffer based on the position of the head. */ if (buf->snapshot) handle->head += to_read; diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index 8debd4f40f06..4c4cbd1f7258 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -546,9 +546,8 @@ static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev, /* * In snapshot mode we simply increment the head by the number of byte - * that were written. User space function cs_etm_find_snapshot() will - * figure out how many bytes to get from the AUX buffer based on the - * position of the head. + * that were written. User space will figure out how many bytes to get + * from the AUX buffer based on the position of the head. */ if (buf->snapshot) handle->head += to_read; diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 14a86a232a18..d6da38e7c682 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1556,9 +1556,8 @@ tmc_update_etr_buffer(struct coresight_device *csdev, /* * In snapshot mode we simply increment the head by the number of byte - * that were written. User space function cs_etm_find_snapshot() will - * figure out how many bytes to get from the AUX buffer based on the - * position of the head. + * that were written. User space will figure out how many bytes to get + * from the AUX buffer based on the position of the head. */ if (etr_perf->snapshot) handle->head += size; From 0abd076217a39c4abc47dcd84d0c8f491f87cbe7 Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Sun, 5 Sep 2021 11:21:44 +0800 Subject: [PATCH 13/39] coresight: tmc-etr: Speed up for bounce buffer in flat mode The AUX bounce buffer is allocated with API dma_alloc_coherent(), in the low level's architecture code, e.g. for Arm64, it maps the memory with the attribution "Normal non-cacheable"; this can be concluded from the definition for pgprot_dmacoherent() in arch/arm64/include/asm/pgtable.h. Later when access the AUX bounce buffer, since the memory mapping is non-cacheable, it's low efficiency due to every load instruction must reach out DRAM. This patch changes to allocate pages with dma_alloc_noncoherent(), the driver can access the memory via cacheable mapping; therefore, load instructions can fetch data from cache lines rather than always read data from DRAM, the driver can boost memory performance. After using the cacheable mapping, the driver uses dma_sync_single_for_cpu() to invalidate cacheline prior to read bounce buffer so can avoid read stale trace data. By measurement the duration for function tmc_update_etr_buffer() with ftrace function_graph tracer, it shows the performance significant improvement for copying 4MiB data from bounce buffer: # echo tmc_etr_get_data_flat_buf > set_graph_notrace // avoid noise # echo tmc_update_etr_buffer > set_graph_function # echo function_graph > current_tracer before: # CPU DURATION FUNCTION CALLS # | | | | | | | 2) | tmc_update_etr_buffer() { ... 2) # 8148.320 us | } after: # CPU DURATION FUNCTION CALLS # | | | | | | | 2) | tmc_update_etr_buffer() { ... 2) # 2525.420 us | } Signed-off-by: Leo Yan Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210905032144.966766-1-leo.yan@linaro.org Signed-off-by: Mathieu Poirier --- .../hwtracing/coresight/coresight-tmc-etr.c | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index d6da38e7c682..867ad8bb9b0c 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -607,8 +607,9 @@ static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata, if (!flat_buf) return -ENOMEM; - flat_buf->vaddr = dma_alloc_coherent(real_dev, etr_buf->size, - &flat_buf->daddr, GFP_KERNEL); + flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size, + &flat_buf->daddr, + DMA_FROM_DEVICE, GFP_KERNEL); if (!flat_buf->vaddr) { kfree(flat_buf); return -ENOMEM; @@ -629,14 +630,18 @@ static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf) if (flat_buf && flat_buf->daddr) { struct device *real_dev = flat_buf->dev->parent; - dma_free_coherent(real_dev, flat_buf->size, - flat_buf->vaddr, flat_buf->daddr); + dma_free_noncoherent(real_dev, etr_buf->size, + flat_buf->vaddr, flat_buf->daddr, + DMA_FROM_DEVICE); } kfree(flat_buf); } static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp) { + struct etr_flat_buf *flat_buf = etr_buf->private; + struct device *real_dev = flat_buf->dev->parent; + /* * Adjust the buffer to point to the beginning of the trace data * and update the available trace data. @@ -646,6 +651,19 @@ static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp) etr_buf->len = etr_buf->size; else etr_buf->len = rwp - rrp; + + /* + * The driver always starts tracing at the beginning of the buffer, + * the only reason why we would get a wrap around is when the buffer + * is full. Sync the entire buffer in one go for this case. + */ + if (etr_buf->offset + etr_buf->len > etr_buf->size) + dma_sync_single_for_cpu(real_dev, flat_buf->daddr, + etr_buf->size, DMA_FROM_DEVICE); + else + dma_sync_single_for_cpu(real_dev, + flat_buf->daddr + etr_buf->offset, + etr_buf->len, DMA_FROM_DEVICE); } static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf, From 8c60acbcb982bfff14d1e85094474671c4f3d006 Mon Sep 17 00:00:00 2001 From: James Clark Date: Wed, 22 Sep 2021 13:51:44 +0100 Subject: [PATCH 14/39] coresight: Don't immediately close events that are run on invalid CPU/sink combos When a traced process runs on a CPU that can't reach the selected sink, the event will be stopped with PERF_HES_STOPPED. This means that even if the process migrates to a valid CPU, tracing will not resume. This can be reproduced (on N1SDP) by using taskset to start the process on CPU 0, and then switching it to CPU 2 (ETF 1 is only reachable from CPU 2): taskset --cpu-list 0 ./perf record -e cs_etm/@tmc_etf1/ --per-thread -- taskset --cpu-list 2 ls This produces a single 0 length AUX record, and then no more trace: 0x3c8 [0x30]: PERF_RECORD_AUX offset: 0 size: 0 flags: 0x1 [T] After the fix, the same command produces normal AUX records. The perf self test "89: Check Arm CoreSight trace data recording and synthesized samples" no longer fails intermittently. This was because the taskset in the test is after the fork, so there is a period where the task is scheduled on a random CPU rather than forced to a valid one. Specifically selecting an invalid CPU will still result in a failure to open the event because it will never produce trace: ./perf record -C 2 -e cs_etm/@tmc_etf0/ failed to mmap with 12 (Cannot allocate memory) The only scenario that has changed is if the CPU mask has a valid CPU sink combo in it. Testing ======= * Coresight self test passes consistently: ./perf test Coresight * CPU wide mode still produces trace: ./perf record -e cs_etm// -a * Invalid -C options still fail to open: ./perf record -C 2,3 -e cs_etm/@tmc_etf0/ failed to mmap with 12 (Cannot allocate memory) * Migrating a task to a valid sink/CPU now produces trace: taskset --cpu-list 0 ./perf record -e cs_etm/@tmc_etf1/ --per-thread -- taskset --cpu-list 2 ls * If the task remains on an invalid CPU, no trace is emitted: taskset --cpu-list 0 ./perf record -e cs_etm/@tmc_etf1/ --per-thread -- ls Reviewed-by: Suzuki K Poulose Signed-off-by: James Clark Link: https://lore.kernel.org/r/20210922125144.133872-2-james.clark@arm.com Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier --- .../hwtracing/coresight/coresight-etm-perf.c | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index 8ebd728d3a80..3bce1ec4c4bf 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -452,9 +452,14 @@ static void etm_event_start(struct perf_event *event, int flags) * sink from this ETM. We can't do much in this case if * the sink was specified or hinted to the driver. For * now, simply don't record anything on this ETM. + * + * As such we pretend that everything is fine, and let + * it continue without actually tracing. The event could + * continue tracing when it moves to a CPU where it is + * reachable to a sink. */ if (!cpumask_test_cpu(cpu, &event_data->mask)) - goto fail_end_stop; + goto out; path = etm_event_cpu_path(event_data, cpu); /* We need a sink, no need to continue without one */ @@ -466,16 +471,15 @@ static void etm_event_start(struct perf_event *event, int flags) if (coresight_enable_path(path, CS_MODE_PERF, handle)) goto fail_end_stop; - /* Tell the perf core the event is alive */ - event->hw.state = 0; - /* Finally enable the tracer */ if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF)) goto fail_disable_path; +out: + /* Tell the perf core the event is alive */ + event->hw.state = 0; /* Save the event_data for this ETM */ ctxt->event_data = event_data; -out: return; fail_disable_path: @@ -485,7 +489,7 @@ fail_end_stop: perf_aux_output_end(handle, 0); fail: event->hw.state = PERF_HES_STOPPED; - goto out; + return; } static void etm_event_stop(struct perf_event *event, int mode) @@ -517,6 +521,19 @@ static void etm_event_stop(struct perf_event *event, int mode) if (WARN_ON(!event_data)) return; + /* + * Check if this ETM was allowed to trace, as decided at + * etm_setup_aux(). If it wasn't allowed to trace, then + * nothing needs to be torn down other than outputting a + * zero sized record. + */ + if (handle->event && (mode & PERF_EF_UPDATE) && + !cpumask_test_cpu(cpu, &event_data->mask)) { + event->hw.state = PERF_HES_STOPPED; + perf_aux_output_end(handle, 0); + return; + } + if (!csdev) return; From 937d3f58cacf377cab7c32e475e1ffa91d611dce Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 14 Sep 2021 11:26:32 +0100 Subject: [PATCH 15/39] coresight: etm4x: Save restore TRFCR_EL1 When the CPU enters a low power mode, the TRFCR_EL1 contents could be reset. Thus we need to save/restore the TRFCR_EL1 along with the ETM4x registers to allow the tracing. The TRFCR related helpers are in a new header file, as we need to use them for TRBE in the later patches. Cc: Mathieu Poirier Cc: Anshuman Khandual Cc: Mike Leach Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210914102641.1852544-2-suzuki.poulose@arm.com [Fixed cosmetic details] Signed-off-by: Mathieu Poirier --- .../coresight/coresight-etm4x-core.c | 43 +++++++++++++------ drivers/hwtracing/coresight/coresight-etm4x.h | 2 + .../coresight/coresight-self-hosted-trace.h | 24 +++++++++++ 3 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 drivers/hwtracing/coresight/coresight-self-hosted-trace.h diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index e24252eaf8e4..537c0d7ee1ed 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -40,6 +40,7 @@ #include "coresight-etm4x.h" #include "coresight-etm-perf.h" #include "coresight-etm4x-cfg.h" +#include "coresight-self-hosted-trace.h" #include "coresight-syscfg.h" static int boot_enable; @@ -1011,7 +1012,7 @@ static void cpu_enable_tracing(struct etmv4_drvdata *drvdata) if (is_kernel_in_hyp_mode()) trfcr |= TRFCR_EL2_CX; - write_sysreg_s(trfcr, SYS_TRFCR_EL1); + write_trfcr(trfcr); } static void etm4_init_arch_data(void *info) @@ -1554,7 +1555,7 @@ static void etm4_init_trace_id(struct etmv4_drvdata *drvdata) drvdata->trcid = coresight_get_trace_id(drvdata->cpu); } -static int etm4_cpu_save(struct etmv4_drvdata *drvdata) +static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) { int i, ret = 0; struct etmv4_save_state *state; @@ -1693,7 +1694,23 @@ out: return ret; } -static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) +static int etm4_cpu_save(struct etmv4_drvdata *drvdata) +{ + int ret = 0; + + /* Save the TRFCR irrespective of whether the ETM is ON */ + if (drvdata->trfc) + drvdata->save_trfcr = read_trfcr(); + /* + * Save and restore the ETM Trace registers only if + * the ETM is active. + */ + if (local_read(&drvdata->mode) && drvdata->save_state) + ret = __etm4_cpu_save(drvdata); + return ret; +} + +static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) { int i; struct etmv4_save_state *state = drvdata->save_state; @@ -1789,6 +1806,14 @@ static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4_cs_lock(drvdata, csa); } +static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) +{ + if (drvdata->trfc) + write_trfcr(drvdata->save_trfcr); + if (drvdata->state_needs_restore) + __etm4_cpu_restore(drvdata); +} + static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, void *v) { @@ -1800,23 +1825,17 @@ static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, drvdata = etmdrvdata[cpu]; - if (!drvdata->save_state) - return NOTIFY_OK; - if (WARN_ON_ONCE(drvdata->cpu != cpu)) return NOTIFY_BAD; switch (cmd) { case CPU_PM_ENTER: - /* save the state if self-hosted coresight is in use */ - if (local_read(&drvdata->mode)) - if (etm4_cpu_save(drvdata)) - return NOTIFY_BAD; + if (etm4_cpu_save(drvdata)) + return NOTIFY_BAD; break; case CPU_PM_EXIT: case CPU_PM_ENTER_FAILED: - if (drvdata->state_needs_restore) - etm4_cpu_restore(drvdata); + etm4_cpu_restore(drvdata); break; default: return NOTIFY_DONE; diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index e5b79bdb9851..82cba16b73a6 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -921,6 +921,7 @@ struct etmv4_save_state { * @lpoverride: If the implementation can support low-power state over. * @trfc: If the implementation supports Arm v8.4 trace filter controls. * @config: structure holding configuration parameters. + * @save_trfcr: Saved TRFCR_EL1 register during a CPU PM event. * @save_state: State to be preserved across power loss * @state_needs_restore: True when there is context to restore after PM exit * @skip_power_up: Indicates if an implementation can skip powering up @@ -973,6 +974,7 @@ struct etmv4_drvdata { bool lpoverride; bool trfc; struct etmv4_config config; + u64 save_trfcr; struct etmv4_save_state *save_state; bool state_needs_restore; bool skip_power_up; diff --git a/drivers/hwtracing/coresight/coresight-self-hosted-trace.h b/drivers/hwtracing/coresight/coresight-self-hosted-trace.h new file mode 100644 index 000000000000..303d71911870 --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-self-hosted-trace.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Arm v8 Self-Hosted trace support. + * + * Copyright (C) 2021 ARM Ltd. + */ + +#ifndef __CORESIGHT_SELF_HOSTED_TRACE_H +#define __CORESIGHT_SELF_HOSTED_TRACE_H + +#include + +static inline u64 read_trfcr(void) +{ + return read_sysreg_s(SYS_TRFCR_EL1); +} + +static inline void write_trfcr(u64 val) +{ + write_sysreg_s(val, SYS_TRFCR_EL1); + isb(); +} + +#endif /* __CORESIGHT_SELF_HOSTED_TRACE_H */ From 5f6fd1aa8cc147b111af1a833574487a87237dc0 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 14 Sep 2021 11:26:33 +0100 Subject: [PATCH 16/39] coresight: etm4x: Use Trace Filtering controls dynamically The Trace Filtering support (FEAT_TRF) ensures that the ETM can be prohibited from generating any trace for a given EL. This is much stricter knob, than the TRCVICTLR exception level masks, which doesn't prevent the ETM from generating Context packets for an "excluded" EL. At the moment, we do a onetime enable trace at user and kernel and leave it untouched for the kernel life time. This implies that the ETM could potentially generate trace packets containing the kernel addresses, and thus leaking the kernel virtual address in the trace. This patch makes the switch dynamic, by honoring the filters set by the user and enforcing them in the TRFCR controls. We also rename the cpu_enable_tracing() appropriately to cpu_detect_trace_filtering() and the drvdata member trfc => trfcr to indicate the "value" of the TRFCR_EL1. Cc: Mathieu Poirier Cc: Al Grant Cc: Mike Leach Cc: Leo Yan Signed-off-by: Suzuki K Poulose Reviewed-by: Anshuman Khandual Link: https://lore.kernel.org/r/20210914102641.1852544-3-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- .../coresight/coresight-etm4x-core.c | 63 ++++++++++++++----- drivers/hwtracing/coresight/coresight-etm4x.h | 7 ++- .../coresight/coresight-self-hosted-trace.h | 7 +++ 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 537c0d7ee1ed..d9a85ca25b60 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -239,6 +239,45 @@ struct etm4_enable_arg { int rc; }; +/* + * etm4x_prohibit_trace - Prohibit the CPU from tracing at all ELs. + * When the CPU supports FEAT_TRF, we could move the ETM to a trace + * prohibited state by filtering the Exception levels via TRFCR_EL1. + */ +static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata) +{ + /* If the CPU doesn't support FEAT_TRF, nothing to do */ + if (!drvdata->trfcr) + return; + cpu_prohibit_trace(); +} + +/* + * etm4x_allow_trace - Allow CPU tracing in the respective ELs, + * as configured by the drvdata->config.mode for the current + * session. Even though we have TRCVICTLR bits to filter the + * trace in the ELs, it doesn't prevent the ETM from generating + * a packet (e.g, TraceInfo) that might contain the addresses from + * the excluded levels. Thus we use the additional controls provided + * via the Trace Filtering controls (FEAT_TRF) to make sure no trace + * is generated for the excluded ELs. + */ +static void etm4x_allow_trace(struct etmv4_drvdata *drvdata) +{ + u64 trfcr = drvdata->trfcr; + + /* If the CPU doesn't support FEAT_TRF, nothing to do */ + if (!trfcr) + return; + + if (drvdata->config.mode & ETM_MODE_EXCL_KERN) + trfcr &= ~TRFCR_ELx_ExTRE; + if (drvdata->config.mode & ETM_MODE_EXCL_USER) + trfcr &= ~TRFCR_ELx_E0TRE; + + write_trfcr(trfcr); +} + #ifdef CONFIG_ETM4X_IMPDEF_FEATURE #define HISI_HIP08_AMBA_ID 0x000b6d01 @@ -443,6 +482,7 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) if (etm4x_is_ete(drvdata)) etm4x_relaxed_write32(csa, TRCRSR_TA, TRCRSR); + etm4x_allow_trace(drvdata); /* Enable the trace unit */ etm4x_relaxed_write32(csa, 1, TRCPRGCTLR); @@ -738,7 +778,6 @@ static int etm4_enable(struct coresight_device *csdev, static void etm4_disable_hw(void *info) { u32 control; - u64 trfcr; struct etmv4_drvdata *drvdata = info; struct etmv4_config *config = &drvdata->config; struct coresight_device *csdev = drvdata->csdev; @@ -765,12 +804,7 @@ static void etm4_disable_hw(void *info) * If the CPU supports v8.4 Trace filter Control, * set the ETM to trace prohibited region. */ - if (drvdata->trfc) { - trfcr = read_sysreg_s(SYS_TRFCR_EL1); - write_sysreg_s(trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE), - SYS_TRFCR_EL1); - isb(); - } + etm4x_prohibit_trace(drvdata); /* * Make sure everything completes before disabling, as recommended * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register, @@ -786,9 +820,6 @@ static void etm4_disable_hw(void *info) if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) dev_err(etm_dev, "timeout while waiting for PM stable Trace Status\n"); - if (drvdata->trfc) - write_sysreg_s(trfcr, SYS_TRFCR_EL1); - /* read the status of the single shot comparators */ for (i = 0; i < drvdata->nr_ss_cmp; i++) { config->ss_status[i] = @@ -990,15 +1021,15 @@ static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, return false; } -static void cpu_enable_tracing(struct etmv4_drvdata *drvdata) +static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata) { u64 dfr0 = read_sysreg(id_aa64dfr0_el1); u64 trfcr; + drvdata->trfcr = 0; if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRACE_FILT_SHIFT)) return; - drvdata->trfc = true; /* * If the CPU supports v8.4 SelfHosted Tracing, enable * tracing at the kernel EL and EL0, forcing to use the @@ -1012,7 +1043,7 @@ static void cpu_enable_tracing(struct etmv4_drvdata *drvdata) if (is_kernel_in_hyp_mode()) trfcr |= TRFCR_EL2_CX; - write_trfcr(trfcr); + drvdata->trfcr = trfcr; } static void etm4_init_arch_data(void *info) @@ -1203,7 +1234,7 @@ static void etm4_init_arch_data(void *info) /* NUMCNTR, bits[30:28] number of counters available for tracing */ drvdata->nr_cntr = BMVAL(etmidr5, 28, 30); etm4_cs_lock(drvdata, csa); - cpu_enable_tracing(drvdata); + cpu_detect_trace_filtering(drvdata); } static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config) @@ -1699,7 +1730,7 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) int ret = 0; /* Save the TRFCR irrespective of whether the ETM is ON */ - if (drvdata->trfc) + if (drvdata->trfcr) drvdata->save_trfcr = read_trfcr(); /* * Save and restore the ETM Trace registers only if @@ -1808,7 +1839,7 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) { - if (drvdata->trfc) + if (drvdata->trfcr) write_trfcr(drvdata->save_trfcr); if (drvdata->state_needs_restore) __etm4_cpu_restore(drvdata); diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 82cba16b73a6..3c4d69b096ca 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -919,7 +919,10 @@ struct etmv4_save_state { * @nooverflow: Indicate if overflow prevention is supported. * @atbtrig: If the implementation can support ATB triggers * @lpoverride: If the implementation can support low-power state over. - * @trfc: If the implementation supports Arm v8.4 trace filter controls. + * @trfcr: If the CPU supports FEAT_TRF, value of the TRFCR_ELx that + * allows tracing at all ELs. We don't want to compute this + * at runtime, due to the additional setting of TRFCR_CX when + * in EL2. Otherwise, 0. * @config: structure holding configuration parameters. * @save_trfcr: Saved TRFCR_EL1 register during a CPU PM event. * @save_state: State to be preserved across power loss @@ -972,7 +975,7 @@ struct etmv4_drvdata { bool nooverflow; bool atbtrig; bool lpoverride; - bool trfc; + u64 trfcr; struct etmv4_config config; u64 save_trfcr; struct etmv4_save_state *save_state; diff --git a/drivers/hwtracing/coresight/coresight-self-hosted-trace.h b/drivers/hwtracing/coresight/coresight-self-hosted-trace.h index 303d71911870..23f05df3f173 100644 --- a/drivers/hwtracing/coresight/coresight-self-hosted-trace.h +++ b/drivers/hwtracing/coresight/coresight-self-hosted-trace.h @@ -21,4 +21,11 @@ static inline void write_trfcr(u64 val) isb(); } +static inline void cpu_prohibit_trace(void) +{ + u64 trfcr = read_trfcr(); + + /* Prohibit tracing at EL0 & the kernel EL */ + write_trfcr(trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE)); +} #endif /* __CORESIGHT_SELF_HOSTED_TRACE_H */ From 2ef43054bb26cd3df53a32f50d7eea796a578931 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 14 Sep 2021 11:26:34 +0100 Subject: [PATCH 17/39] coresight: etm-pmu: Ensure the AUX handle is valid The ETM perf infrastructure closes out a handle during event_stop or on an error in starting the event. In either case, it is possible for a "sink" to update/close the handle, under certain circumstances. (e.g no space in ring buffer.). So, ensure that we handle this gracefully in the PMU driver by verifying the handle is still valid. Cc: Mathieu Poirier Cc: Mike Leach Cc: Anshuman Khandual Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210914102641.1852544-4-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- .../hwtracing/coresight/coresight-etm-perf.c | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index 3bce1ec4c4bf..c039b6ae206f 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -485,8 +485,15 @@ out: fail_disable_path: coresight_disable_path(path); fail_end_stop: - perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); - perf_aux_output_end(handle, 0); + /* + * Check if the handle is still associated with the event, + * to handle cases where if the sink failed to start the + * trace and TRUNCATED the handle already. + */ + if (READ_ONCE(handle->event)) { + perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + perf_aux_output_end(handle, 0); + } fail: event->hw.state = PERF_HES_STOPPED; return; @@ -567,7 +574,21 @@ static void etm_event_stop(struct perf_event *event, int mode) size = sink_ops(sink)->update_buffer(sink, handle, event_data->snk_config); - perf_aux_output_end(handle, size); + /* + * Make sure the handle is still valid as the + * sink could have closed it from an IRQ. + * The sink driver must handle the race with + * update_buffer() and IRQ. Thus either we + * should get a valid handle and valid size + * (which may be 0). + * + * But we should never get a non-zero size with + * an invalid handle. + */ + if (READ_ONCE(handle->event)) + perf_aux_output_end(handle, size); + else + WARN_ON(size); } /* Disabling the path make its elements available to other sessions */ From 5bd9ff830c8728bf6a532ff893b73e7afae007f1 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 14 Sep 2021 11:26:35 +0100 Subject: [PATCH 18/39] coresight: trbe: Ensure the format flag is always set When the TRBE is stopped on truncating an event, we may not set the FORMAT flag, even though the size of the record is 0. Let us be consistent and not confuse the user. To ensure that the format flag is always set on all the records generated by TRBE, set the flag when we have a new handle. Rather than deferring to the "end" operation, which makes it clear. So, we can do this from - arm_trbe_enable() -> When a new handle is provided by the CoreSight PMU, triggered via etm_event_start() - trbe_handle_overflow() -> When we begin a new handle after closing the previous on overflow. Cc: Anshuman Khandual Cc: Leo Yan Cc: Mike Leach Cc: Mathieu Poirier Signed-off-by: Suzuki K Poulose Reviewed-by: Anshuman Khandual Link: https://lore.kernel.org/r/20210914102641.1852544-5-suzuki.poulose@arm.com [Fixed inverted words in title] Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 176868496879..81bf183a73a1 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -554,8 +554,6 @@ static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev, if (cpudata->mode != CS_MODE_PERF) return 0; - perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW); - /* * We are about to disable the TRBE. And this could in turn * fill up the buffer triggering, an IRQ. This could be consumed @@ -648,6 +646,7 @@ static int arm_trbe_enable(struct coresight_device *csdev, u32 mode, void *data) if (mode != CS_MODE_PERF) return -EINVAL; + perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW); *this_cpu_ptr(drvdata->handle) = handle; cpudata->buf = buf; cpudata->mode = mode; @@ -710,8 +709,7 @@ static void trbe_handle_overflow(struct perf_output_handle *handle) * Mark the buffer as truncated, as we have stopped the trace * collection upon the WRAP event, without stopping the source. */ - perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW | - PERF_AUX_FLAG_TRUNCATED); + perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); perf_aux_output_end(handle, size); event_data = perf_aux_output_begin(handle, event); if (!event_data) { @@ -725,6 +723,7 @@ static void trbe_handle_overflow(struct perf_output_handle *handle) *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL; return; } + perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW); buf->trbe_limit = compute_trbe_buffer_limit(handle); buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); if (buf->trbe_limit == buf->trbe_base) { From acee3ef86d5cfda094fa9806935aee45833444e3 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 14 Sep 2021 11:26:36 +0100 Subject: [PATCH 19/39] coresight: trbe: Drop duplicate TRUNCATE flags We mark the buffer as TRUNCATED when there is no space left in the buffer. But we do it at different points. __trbe_normal_offset() and also, at all the callers of the above function via compute_trbe_buffer_limit(), when the limit == base (i.e offset = 0 as returned by the __trbe_normal_offset()). So, given that the callers already mark the buffer as TRUNCATED drop the caller inside the __trbe_normal_offset(). This is in preparation to moving the handling of TRUNCATED into a central place. Cc: Anshuman Khandual Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Signed-off-by: Suzuki K Poulose Reviewed-by: Anshuman Khandual Link: https://lore.kernel.org/r/20210914102641.1852544-6-suzuki.poulose@arm.com [Moved comment as Anshuman requested] Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 81bf183a73a1..4479e689c38a 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -252,13 +252,9 @@ static unsigned long __trbe_normal_offset(struct perf_output_handle *handle) * trbe_base trbe_base + nr_pages * * Perf aux buffer does not have any space for the driver to write into. - * Just communicate trace truncation event to the user space by marking - * it with PERF_AUX_FLAG_TRUNCATED. */ - if (!handle->size) { - perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + if (!handle->size) return 0; - } /* Compute the tail and wakeup indices now that we've aligned head */ tail = PERF_IDX2OFF(handle->head + handle->size, buf); @@ -360,7 +356,6 @@ static unsigned long __trbe_normal_offset(struct perf_output_handle *handle) return limit; trbe_pad_buf(handle, handle->size); - perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); return 0; } @@ -688,6 +683,11 @@ static void trbe_handle_spurious(struct perf_output_handle *handle) buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); if (buf->trbe_limit == buf->trbe_base) { trbe_drain_and_disable_local(); + /* + * Just communicate trace truncation event to the user space by + * marking it with PERF_AUX_FLAG_TRUNCATED. + */ + perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); return; } trbe_enable_hw(buf); From 04a37a174e5697d90c129c286bdb6bbcb516058f Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 14 Sep 2021 11:26:39 +0100 Subject: [PATCH 20/39] coresight: trbe: Unify the enabling sequence Unify the sequence of enabling the TRBE. We do this from event_start and also from the TRBE IRQ handler. Lets move this to a common helper. The only minor functional change is returning an error when we fail to enable the TRBE. This should be handled already. Since we now have unique entry point to trying to enable TRBE, move the format flag setting to the central place. Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210914102641.1852544-9-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 37 ++++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 4479e689c38a..2cf28de2e447 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -629,6 +629,21 @@ done: return size; } +static int __arm_trbe_enable(struct trbe_buf *buf, + struct perf_output_handle *handle) +{ + perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW); + buf->trbe_limit = compute_trbe_buffer_limit(handle); + buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); + if (buf->trbe_limit == buf->trbe_base) { + trbe_stop_and_truncate_event(handle); + return -ENOSPC; + } + *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle; + trbe_enable_hw(buf); + return 0; +} + static int arm_trbe_enable(struct coresight_device *csdev, u32 mode, void *data) { struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); @@ -641,19 +656,11 @@ static int arm_trbe_enable(struct coresight_device *csdev, u32 mode, void *data) if (mode != CS_MODE_PERF) return -EINVAL; - perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW); - *this_cpu_ptr(drvdata->handle) = handle; cpudata->buf = buf; cpudata->mode = mode; buf->cpudata = cpudata; - buf->trbe_limit = compute_trbe_buffer_limit(handle); - buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); - if (buf->trbe_limit == buf->trbe_base) { - trbe_stop_and_truncate_event(handle); - return 0; - } - trbe_enable_hw(buf); - return 0; + + return __arm_trbe_enable(buf, handle); } static int arm_trbe_disable(struct coresight_device *csdev) @@ -723,15 +730,7 @@ static void trbe_handle_overflow(struct perf_output_handle *handle) *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL; return; } - perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW); - buf->trbe_limit = compute_trbe_buffer_limit(handle); - buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); - if (buf->trbe_limit == buf->trbe_base) { - trbe_stop_and_truncate_event(handle); - return; - } - *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle; - trbe_enable_hw(buf); + __arm_trbe_enable(buf, handle); } static bool is_perf_trbe(struct perf_output_handle *handle) From 85fb92353e0dece18048bdb35c7e6b682129deff Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Thu, 23 Sep 2021 15:39:15 +0100 Subject: [PATCH 21/39] coresight: trbe: irq handler: Do not disable TRBE if no action is needed The IRQ handler of the TRBE driver could race against the update_buffer() in consuming the IRQ. So, if the update_buffer() gets to processing the TRBE irq, the TRBSR will be cleared. Thus by the time IRQ handler is triggered, there is nothing to do there. Handle these cases and do not disable the TRBE unnecessarily. Since the TRBSR can be read without stopping the TRBE, we can check that before disabling the TRBE. Cc: Mathieu Poirier Cc: Anshuman Khandual Cc: Leo Yan Cc: Mike Leach Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210923143919.2944311-2-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 2cf28de2e447..e78800ba5b5b 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -762,12 +762,7 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) enum trbe_fault_action act; u64 status; - /* - * Ensure the trace is visible to the CPUs and - * any external aborts have been resolved. - */ - trbe_drain_and_disable_local(); - + /* Reads to TRBSR_EL1 is fine when TRBE is active */ status = read_sysreg_s(SYS_TRBSR_EL1); /* * If the pending IRQ was handled by update_buffer callback @@ -776,6 +771,11 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) if (!is_trbe_irq(status)) return IRQ_NONE; + /* + * Ensure the trace is visible to the CPUs and + * any external aborts have been resolved. + */ + trbe_drain_and_disable_local(); clr_trbe_irq(); isb(); From 7037a39d379733f5352af5e04a202dad35562ef3 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Thu, 23 Sep 2021 15:39:16 +0100 Subject: [PATCH 22/39] coresight: trbe: Fix handling of spurious interrupts On a spurious IRQ, right now we disable the TRBE and then re-enable it back, resetting the "buffer" pointers(i.e BASE, LIMIT and more importantly WRITE) to the original pointers from the AUX handle. This implies that we overwrite any trace that was written so far, (by overwriting TRBPTR) while we should have ignored the IRQ. On detecting a spurious IRQ after examining the TRBSR we simply re-enable the TRBE without touching the other parameters. Cc: Anshuman Khandual Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210923143919.2944311-3-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 22 ++++++++------------ 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index e78800ba5b5b..47120d263639 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -684,20 +684,16 @@ static int arm_trbe_disable(struct coresight_device *csdev) static void trbe_handle_spurious(struct perf_output_handle *handle) { - struct trbe_buf *buf = etm_perf_sink_config(handle); + u64 limitr = read_sysreg_s(SYS_TRBLIMITR_EL1); - buf->trbe_limit = compute_trbe_buffer_limit(handle); - buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); - if (buf->trbe_limit == buf->trbe_base) { - trbe_drain_and_disable_local(); - /* - * Just communicate trace truncation event to the user space by - * marking it with PERF_AUX_FLAG_TRUNCATED. - */ - perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); - return; - } - trbe_enable_hw(buf); + /* + * If the IRQ was spurious, simply re-enable the TRBE + * back without modifying the buffer parameters to + * retain the trace collected so far. + */ + limitr |= TRBLIMITR_ENABLE; + write_sysreg_s(limitr, SYS_TRBLIMITR_EL1); + isb(); } static void trbe_handle_overflow(struct perf_output_handle *handle) From 0a5f355633eaacb7300598d96e8013a401a02c5e Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Thu, 23 Sep 2021 15:39:17 +0100 Subject: [PATCH 23/39] coresight: trbe: Do not truncate buffer on IRQ The TRBE driver marks the AUX buffer as TRUNCATED when we get an IRQ on FILL event. This has rather unwanted side-effect of the event being disabled when there may be more space in the ring buffer. So, instead of TRUNCATE we need a different flag to indicate that the trace may have lost a few bytes (i.e from the point of generating the FILL event until the IRQ is consumed). Anyways, the userspace must use the size from RECORD_AUX headers to restrict the "trace" decoding. Using PARTIAL flag causes the perf tool to generate the following warning: Warning: AUX data had gaps in it XX times out of YY! Are you running a KVM guest in the background? which is pointlessly scary for a user. The other remaining options are : - COLLISION - Use by SPE to indicate samples collided - Add a new flag - Specifically for CoreSight, doesn't sound so good, if we can re-use something. Given that we don't already use the "COLLISION" flag, the above behavior can be notified using this flag for CoreSight. Cc: Mathieu Poirier Cc: James Clark Cc: Mike Leach Cc: Anshuman Khandual Cc: Leo Yan Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210923143919.2944311-4-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 27 +++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 47120d263639..0a9106c15639 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -120,6 +120,25 @@ static void trbe_reset_local(void) write_sysreg_s(0, SYS_TRBSR_EL1); } +static void trbe_report_wrap_event(struct perf_output_handle *handle) +{ + /* + * Mark the buffer to indicate that there was a WRAP event by + * setting the COLLISION flag. This indicates to the user that + * the TRBE trace collection was stopped without stopping the + * ETE and thus there might be some amount of trace that was + * lost between the time the WRAP was detected and the IRQ + * was consumed by the CPU. + * + * Setting the TRUNCATED flag would move the event to STOPPED + * state unnecessarily, even when there is space left in the + * ring buffer. Using the COLLISION flag doesn't have this side + * effect. We only set TRUNCATED flag when there is no space + * left in the ring buffer. + */ + perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION); +} + static void trbe_stop_and_truncate_event(struct perf_output_handle *handle) { struct trbe_buf *buf = etm_perf_sink_config(handle); @@ -612,7 +631,7 @@ static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev, * for correct size. Also, mark the buffer truncated. */ write = get_trbe_limit_pointer(); - perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + trbe_report_wrap_event(handle); } offset = write - base; @@ -708,11 +727,7 @@ static void trbe_handle_overflow(struct perf_output_handle *handle) if (buf->snapshot) handle->head += size; - /* - * Mark the buffer as truncated, as we have stopped the trace - * collection upon the WRAP event, without stopping the source. - */ - perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + trbe_report_wrap_event(handle); perf_aux_output_end(handle, size); event_data = perf_aux_output_begin(handle, event); if (!event_data) { From 9bef9d0850a024a07f46d13828c05becf03eb79f Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Thu, 23 Sep 2021 15:39:18 +0100 Subject: [PATCH 24/39] coresight: trbe: End the AUX handle on truncation When we detect that there isn't enough space left to start a meaningful session, we disable the TRBE, marking the buffer as TRUNCATED. But we delay the notification to the perf layer by perf_aux_output_end() until the event is scheduled out, triggered from the kernel perf layer. This will cause significant black outs in the trace. Now that the CoreSight PMU layer can handle a closed "AUX" handle properly, we can close the handle as soon as we detect the case, allowing the userspace to collect and re-enable the event. Also, while in the IRQ handler, move the irq_work_run() after we have updated the handle, to make sure the "TRUNCATED" flag causes the event to be disabled as soon as possible. Cc: Anshuman Khandual Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Cc: Peter Zijlstra (Intel) Cc: Will Deacon Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210923143919.2944311-5-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 26 ++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 0a9106c15639..4174300f1344 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -152,6 +152,7 @@ static void trbe_stop_and_truncate_event(struct perf_output_handle *handle) */ trbe_drain_and_disable_local(); perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + perf_aux_output_end(handle, 0); *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL; } @@ -715,7 +716,7 @@ static void trbe_handle_spurious(struct perf_output_handle *handle) isb(); } -static void trbe_handle_overflow(struct perf_output_handle *handle) +static int trbe_handle_overflow(struct perf_output_handle *handle) { struct perf_event *event = handle->event; struct trbe_buf *buf = etm_perf_sink_config(handle); @@ -739,9 +740,10 @@ static void trbe_handle_overflow(struct perf_output_handle *handle) */ trbe_drain_and_disable_local(); *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL; - return; + return -EINVAL; } - __arm_trbe_enable(buf, handle); + + return __arm_trbe_enable(buf, handle); } static bool is_perf_trbe(struct perf_output_handle *handle) @@ -772,6 +774,7 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) struct perf_output_handle *handle = *handle_ptr; enum trbe_fault_action act; u64 status; + bool truncated = false; /* Reads to TRBSR_EL1 is fine when TRBE is active */ status = read_sysreg_s(SYS_TRBSR_EL1); @@ -796,24 +799,27 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) if (!is_perf_trbe(handle)) return IRQ_NONE; - /* - * Ensure perf callbacks have completed, which may disable - * the trace buffer in response to a TRUNCATION flag. - */ - irq_work_run(); - act = trbe_get_fault_act(status); switch (act) { case TRBE_FAULT_ACT_WRAP: - trbe_handle_overflow(handle); + truncated = !!trbe_handle_overflow(handle); break; case TRBE_FAULT_ACT_SPURIOUS: trbe_handle_spurious(handle); break; case TRBE_FAULT_ACT_FATAL: trbe_stop_and_truncate_event(handle); + truncated = true; break; } + + /* + * If the buffer was truncated, ensure perf callbacks + * have completed, which will disable the event. + */ + if (truncated) + irq_work_run(); + return IRQ_HANDLED; } From dcfecfa444b12546c028e0f5a66255347a78cbd2 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Thu, 23 Sep 2021 15:39:19 +0100 Subject: [PATCH 25/39] coresight: trbe: Prohibit trace before disabling TRBE When the TRBE generates an IRQ, we stop the TRBE, collect the trace and then reprogram the TRBE with the updated buffer pointers, whenever possible. We might also leave the TRBE disabled, if there is not enough space left in the buffer. However, we do not touch the ETE at all during all of this. This means the ETE is only disabled when the event is disabled later (via irq_work). This is incorrect, as the ETE trace is still ON without actually being captured and may be routed to the ATB (even if it is for a short duration). So, we move the CPU into trace prohibited state always before disabling the TRBE, upon entering the IRQ handler. The state is restored if the TRBE is enabled back. Otherwise the trace remains prohibited. Since, the ETM/ETE driver now controls the TRFCR_EL1 per session, the tracing can be restored/enabled back when the event is rescheduled in. Cc: Anshuman Khandual Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20210923143919.2944311-6-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- .../hwtracing/coresight/coresight-self-hosted-trace.h | 4 +++- drivers/hwtracing/coresight/coresight-trbe.c | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/coresight-self-hosted-trace.h b/drivers/hwtracing/coresight/coresight-self-hosted-trace.h index 23f05df3f173..53840a2c41f2 100644 --- a/drivers/hwtracing/coresight/coresight-self-hosted-trace.h +++ b/drivers/hwtracing/coresight/coresight-self-hosted-trace.h @@ -21,11 +21,13 @@ static inline void write_trfcr(u64 val) isb(); } -static inline void cpu_prohibit_trace(void) +static inline u64 cpu_prohibit_trace(void) { u64 trfcr = read_trfcr(); /* Prohibit tracing at EL0 & the kernel EL */ write_trfcr(trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE)); + /* Return the original value of the TRFCR */ + return trfcr; } #endif /* __CORESIGHT_SELF_HOSTED_TRACE_H */ diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 4174300f1344..a53ee98f312f 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -16,6 +16,7 @@ #define pr_fmt(fmt) DRVNAME ": " fmt #include +#include "coresight-self-hosted-trace.h" #include "coresight-trbe.h" #define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT)) @@ -775,6 +776,7 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) enum trbe_fault_action act; u64 status; bool truncated = false; + u64 trfcr; /* Reads to TRBSR_EL1 is fine when TRBE is active */ status = read_sysreg_s(SYS_TRBSR_EL1); @@ -785,6 +787,8 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) if (!is_trbe_irq(status)) return IRQ_NONE; + /* Prohibit the CPU from tracing before we disable the TRBE */ + trfcr = cpu_prohibit_trace(); /* * Ensure the trace is visible to the CPUs and * any external aborts have been resolved. @@ -816,9 +820,14 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) /* * If the buffer was truncated, ensure perf callbacks * have completed, which will disable the event. + * + * Otherwise, restore the trace filter controls to + * allow the tracing. */ if (truncated) irq_work_run(); + else + write_trfcr(trfcr); return IRQ_HANDLED; } From 0605b89d0597f326aba3f70d0574750368a9fbe0 Mon Sep 17 00:00:00 2001 From: Tao Zhang Date: Fri, 24 Sep 2021 18:06:20 +0800 Subject: [PATCH 26/39] coresight: etm4x: Add ETM PID for Kryo-5XX Add ETM PID for Kryo-5XX to the list of supported ETMs. Otherwise, Kryo-5XX ETMs will not be initialized successfully. e.g. This change can be verified on qrb5165-rb5 board. ETM4-ETM7 nodes will not be visible without this change. Signed-off-by: Tao Zhang Link: https://lore.kernel.org/r/1632477981-13632-2-git-send-email-quic_taozha@quicinc.com Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index d9a85ca25b60..86a313857b58 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -2149,6 +2149,7 @@ static const struct amba_id etm4_ids[] = { CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */ CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */ CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */ + CS_AMBA_UCI_ID(0x000bbd0d, uci_id_etm4),/* Qualcomm Kryo 5XX Cortex-A77 */ CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */ CS_AMBA_UCI_ID(0x000b6d01, uci_id_etm4),/* HiSilicon-Hip08 */ CS_AMBA_UCI_ID(0x000b6d02, uci_id_etm4),/* HiSilicon-Hip09 */ From bb5293e334af51b19b62d8bef1852ea13e935e9b Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 21 Sep 2021 14:41:05 +0100 Subject: [PATCH 27/39] coresight: trbe: Fix incorrect access of the sink specific data The TRBE driver wrongly treats the aux private data as the TRBE driver specific buffer for a given perf handle, while it is the ETM PMU's event specific data. Fix this by correcting the instance to use appropriate helper. Cc: stable Fixes: 3fbf7f011f24 ("coresight: sink: Add TRBE driver") Signed-off-by: Suzuki K Poulose Reviewed-by: Anshuman Khandual Link: https://lore.kernel.org/r/20210921134121.2423546-2-suzuki.poulose@arm.com [Fixed 13 character SHA down to 12] Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index a53ee98f312f..2825ccb0cf39 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -382,7 +382,7 @@ static unsigned long __trbe_normal_offset(struct perf_output_handle *handle) static unsigned long trbe_normal_offset(struct perf_output_handle *handle) { - struct trbe_buf *buf = perf_get_aux(handle); + struct trbe_buf *buf = etm_perf_sink_config(handle); u64 limit = __trbe_normal_offset(handle); u64 head = PERF_IDX2OFF(handle->head, buf); From a08025b3fe56185290a1ea476581f03ca733f967 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Thu, 14 Oct 2021 15:22:38 +0100 Subject: [PATCH 28/39] coresight: trbe: Defer the probe on offline CPUs If a CPU is offline during the driver init, we could end up causing a kernel crash trying to register the coresight device for the TRBE instance. The trbe_cpudata for the TRBE instance is initialized only when it is probed. Otherwise, we could end up dereferencing a NULL cpudata->drvdata. e.g: [ 0.149999] coresight ete0: CPU0: ete v1.1 initialized [ 0.149999] coresight-etm4x ete_1: ETM arch init failed [ 0.149999] coresight-etm4x: probe of ete_1 failed with error -22 [ 0.150085] Unable to handle kernel NULL pointer dereference at virtual address 0000000000000050 [ 0.150085] Mem abort info: [ 0.150085] ESR = 0x96000005 [ 0.150085] EC = 0x25: DABT (current EL), IL = 32 bits [ 0.150085] SET = 0, FnV = 0 [ 0.150085] EA = 0, S1PTW = 0 [ 0.150085] Data abort info: [ 0.150085] ISV = 0, ISS = 0x00000005 [ 0.150085] CM = 0, WnR = 0 [ 0.150085] [0000000000000050] user address but active_mm is swapper [ 0.150085] Internal error: Oops: 96000005 [#1] PREEMPT SMP [ 0.150085] Modules linked in: [ 0.150085] Hardware name: FVP Base RevC (DT) [ 0.150085] pstate: 00800009 (nzcv daif -PAN +UAO -TCO BTYPE=--) [ 0.150155] pc : arm_trbe_register_coresight_cpu+0x74/0x144 [ 0.150155] lr : arm_trbe_register_coresight_cpu+0x48/0x144 ... [ 0.150237] Call trace: [ 0.150237] arm_trbe_register_coresight_cpu+0x74/0x144 [ 0.150237] arm_trbe_device_probe+0x1c0/0x2d8 [ 0.150259] platform_drv_probe+0x94/0xbc [ 0.150259] really_probe+0x1bc/0x4a8 [ 0.150266] driver_probe_device+0x7c/0xb8 [ 0.150266] device_driver_attach+0x6c/0xac [ 0.150266] __driver_attach+0xc4/0x148 [ 0.150266] bus_for_each_dev+0x7c/0xc8 [ 0.150266] driver_attach+0x24/0x30 [ 0.150266] bus_add_driver+0x100/0x1e0 [ 0.150266] driver_register+0x78/0x110 [ 0.150266] __platform_driver_register+0x44/0x50 [ 0.150266] arm_trbe_init+0x28/0x84 [ 0.150266] do_one_initcall+0x94/0x2bc [ 0.150266] do_initcall_level+0xa4/0x158 [ 0.150266] do_initcalls+0x54/0x94 [ 0.150319] do_basic_setup+0x24/0x30 [ 0.150319] kernel_init_freeable+0xe8/0x14c [ 0.150319] kernel_init+0x14/0x18c [ 0.150319] ret_from_fork+0x10/0x30 [ 0.150319] Code: f94012c8 b0004ce2 9134a442 52819801 (f9402917) [ 0.150319] ---[ end trace d23e0cfe5098535e ]--- [ 0.150346] Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b Fix this by skipping the step, if we are unable to probe the CPU. Fixes: 3fbf7f011f24 ("coresight: sink: Add TRBE driver") Reported-by: Bransilav Rankov Cc: Anshuman Khandual Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Cc: stable Tested-by: Branislav Rankov Signed-off-by: Suzuki K Poulose Reviewed-by: Anshuman Khandual Link: https://lore.kernel.org/r/20211014142238.2221248-1-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 2825ccb0cf39..5d77baba8b0f 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -893,6 +893,10 @@ static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cp if (WARN_ON(trbe_csdev)) return; + /* If the TRBE was not probed on the CPU, we shouldn't be here */ + if (WARN_ON(!cpudata->drvdata)) + return; + dev = &cpudata->drvdata->pdev->dev; desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu); if (!desc.name) @@ -974,7 +978,9 @@ static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata) return -ENOMEM; for_each_cpu(cpu, &drvdata->supported_cpus) { - smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1); + /* If we fail to probe the CPU, let us defer it to hotplug callbacks */ + if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1)) + continue; if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) arm_trbe_register_coresight_cpu(drvdata, cpu); if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) From 41c0e5b7a35392c90a467a9a085178642a131324 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:43 +0100 Subject: [PATCH 29/39] coresight: trbe: Add a helper to calculate the trace generated We collect the trace from the TRBE on FILL event from IRQ context and via update_buffer(), when the event is stopped. Let us consolidate how we calculate the trace generated into a helper. Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-6-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 47 ++++++++++++-------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 5d77baba8b0f..54f5d61ee9fb 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -499,6 +499,29 @@ static enum trbe_fault_action trbe_get_fault_act(u64 trbsr) return TRBE_FAULT_ACT_SPURIOUS; } +static unsigned long trbe_get_trace_size(struct perf_output_handle *handle, + struct trbe_buf *buf, bool wrap) +{ + u64 write; + u64 start_off, end_off; + + /* + * If the TRBE has wrapped around the write pointer has + * wrapped and should be treated as limit. + */ + if (wrap) + write = get_trbe_limit_pointer(); + else + write = get_trbe_write_pointer(); + + end_off = write - get_trbe_base_pointer(); + start_off = PERF_IDX2OFF(handle->head, buf); + + if (WARN_ON_ONCE(end_off < start_off)) + return 0; + return (end_off - start_off); +} + static void *arm_trbe_alloc_buffer(struct coresight_device *csdev, struct perf_event *event, void **pages, int nr_pages, bool snapshot) @@ -560,9 +583,9 @@ static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev, struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); struct trbe_buf *buf = config; enum trbe_fault_action act; - unsigned long size, offset; - unsigned long write, base, status; + unsigned long size, status; unsigned long flags; + bool wrap = false; WARN_ON(buf->cpudata != cpudata); WARN_ON(cpudata->cpu != smp_processor_id()); @@ -602,8 +625,6 @@ static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev, * handle gets freed in etm_event_stop(). */ trbe_drain_and_disable_local(); - write = get_trbe_write_pointer(); - base = get_trbe_base_pointer(); /* Check if there is a pending interrupt and handle it here */ status = read_sysreg_s(SYS_TRBSR_EL1); @@ -627,20 +648,11 @@ static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev, goto done; } - /* - * Otherwise, the buffer is full and the write pointer - * has reached base. Adjust this back to the Limit pointer - * for correct size. Also, mark the buffer truncated. - */ - write = get_trbe_limit_pointer(); trbe_report_wrap_event(handle); + wrap = true; } - offset = write - base; - if (WARN_ON_ONCE(offset < PERF_IDX2OFF(handle->head, buf))) - size = 0; - else - size = offset - PERF_IDX2OFF(handle->head, buf); + size = trbe_get_trace_size(handle, buf, wrap); done: local_irq_restore(flags); @@ -721,11 +733,10 @@ static int trbe_handle_overflow(struct perf_output_handle *handle) { struct perf_event *event = handle->event; struct trbe_buf *buf = etm_perf_sink_config(handle); - unsigned long offset, size; + unsigned long size; struct etm_event_data *event_data; - offset = get_trbe_limit_pointer() - get_trbe_base_pointer(); - size = offset - PERF_IDX2OFF(handle->head, buf); + size = trbe_get_trace_size(handle, buf, true); if (buf->snapshot) handle->head += size; From 4585481af322e3ae53dfb78e3ec046ddfdf8902a Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:44 +0100 Subject: [PATCH 30/39] coresight: trbe: Add a helper to pad a given buffer area Refactor the helper to pad a given AUX buffer area to allow "filling" ignore packets, without moving any handle pointers. This will be useful in working around errata, where we may have to fill the buffer after a session. Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-7-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 54f5d61ee9fb..e3767f21ee68 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -199,12 +199,18 @@ static void trbe_stop_and_truncate_event(struct perf_output_handle *handle) * consumed from the user space. The enabled TRBE buffer area is a moving subset of * the allocated perf auxiliary buffer. */ + +static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len) +{ + memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len); +} + static void trbe_pad_buf(struct perf_output_handle *handle, int len) { struct trbe_buf *buf = etm_perf_sink_config(handle); u64 head = PERF_IDX2OFF(handle->head, buf); - memset((void *)buf->trbe_base + head, ETE_IGNORE_PACKET, len); + __trbe_pad_buf(buf, head, len); if (!buf->snapshot) perf_aux_output_skip(handle, len); } From 2336a7b29b5826a9cd15c39818ded665f88d2e5b Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:45 +0100 Subject: [PATCH 31/39] coresight: trbe: Decouple buffer base from the hardware base We always set the TRBBASER_EL1 to the base of the virtual ring buffer. We are about to change this for working around an erratum. So, in preparation to that, allow the driver to choose a different base for the TRBBASER_EL1 (which is within the buffer range). Cc: Anshuman Khandual Cc: Mike Leach Cc: Mathieu Poirier Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-8-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index e3767f21ee68..ae0bde9630f6 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -57,6 +57,8 @@ struct trbe_buf { * trbe_limit sibling pointers. */ unsigned long trbe_base; + /* The base programmed into the TRBE */ + unsigned long trbe_hw_base; unsigned long trbe_limit; unsigned long trbe_write; int nr_pages; @@ -470,12 +472,13 @@ static void set_trbe_limit_pointer_enabled(unsigned long addr) static void trbe_enable_hw(struct trbe_buf *buf) { - WARN_ON(buf->trbe_write < buf->trbe_base); + WARN_ON(buf->trbe_hw_base < buf->trbe_base); + WARN_ON(buf->trbe_write < buf->trbe_hw_base); WARN_ON(buf->trbe_write >= buf->trbe_limit); set_trbe_disabled(); isb(); clr_trbe_status(); - set_trbe_base_pointer(buf->trbe_base); + set_trbe_base_pointer(buf->trbe_hw_base); set_trbe_write_pointer(buf->trbe_write); /* @@ -520,7 +523,12 @@ static unsigned long trbe_get_trace_size(struct perf_output_handle *handle, else write = get_trbe_write_pointer(); - end_off = write - get_trbe_base_pointer(); + /* + * TRBE may use a different base address than the base + * of the ring buffer. Thus use the beginning of the ring + * buffer to compute the offsets. + */ + end_off = write - buf->trbe_base; start_off = PERF_IDX2OFF(handle->head, buf); if (WARN_ON_ONCE(end_off < start_off)) @@ -678,6 +686,8 @@ static int __arm_trbe_enable(struct trbe_buf *buf, trbe_stop_and_truncate_event(handle); return -ENOSPC; } + /* Set the base of the TRBE to the buffer base */ + buf->trbe_hw_base = buf->trbe_base; *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle; trbe_enable_hw(buf); return 0; @@ -771,7 +781,7 @@ static bool is_perf_trbe(struct perf_output_handle *handle) struct trbe_drvdata *drvdata = cpudata->drvdata; int cpu = smp_processor_id(); - WARN_ON(buf->trbe_base != get_trbe_base_pointer()); + WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer()); WARN_ON(buf->trbe_limit != get_trbe_limit_pointer()); if (cpudata->mode != CS_MODE_PERF) From e4bc8829a748c61ab07617b41da5d071b060126e Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:46 +0100 Subject: [PATCH 32/39] coresight: trbe: Allow driver to choose a different alignment The TRBE hardware mandates a minimum alignment for the TRBPTR_EL1, advertised via the TRBIDR_EL1. This is used by the driver to align the buffer write head. This patch allows the driver to choose a different alignment from that of the hardware, by decoupling the alignment tracking. This will be useful for working around errata. Cc: Mathieu Poirier Cc: Anshuman Khandual Cc: Mike Leach Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-9-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index ae0bde9630f6..cdbb134892f6 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -67,8 +67,18 @@ struct trbe_buf { struct trbe_cpudata *cpudata; }; +/* + * struct trbe_cpudata: TRBE instance specific data + * @trbe_flag - TRBE dirty/access flag support + * @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1. + * @trbe_align - Software alignment used for the TRBPTR_EL1. + * @cpu - CPU this TRBE belongs to. + * @mode - Mode of current operation. (perf/disabled) + * @drvdata - TRBE specific drvdata + */ struct trbe_cpudata { bool trbe_flag; + u64 trbe_hw_align; u64 trbe_align; int cpu; enum cs_mode mode; @@ -875,7 +885,7 @@ static ssize_t align_show(struct device *dev, struct device_attribute *attr, cha { struct trbe_cpudata *cpudata = dev_get_drvdata(dev); - return sprintf(buf, "%llx\n", cpudata->trbe_align); + return sprintf(buf, "%llx\n", cpudata->trbe_hw_align); } static DEVICE_ATTR_RO(align); @@ -967,11 +977,13 @@ static void arm_trbe_probe_cpu(void *info) goto cpu_clear; } - cpudata->trbe_align = 1ULL << get_trbe_address_align(trbidr); - if (cpudata->trbe_align > SZ_2K) { + cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr); + if (cpudata->trbe_hw_align > SZ_2K) { pr_err("Unsupported alignment on cpu %d\n", cpu); goto cpu_clear; } + + cpudata->trbe_align = cpudata->trbe_hw_align; cpudata->trbe_flag = get_trbe_flag_update(trbidr); cpudata->cpu = cpu; cpudata->drvdata = drvdata; From 8a1065127d952e22a95ccf1c4b84e733f656bb2c Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:47 +0100 Subject: [PATCH 33/39] coresight: trbe: Add infrastructure for Errata handling Add a minimal infrastructure to keep track of the errata affecting the given TRBE instance. Given that we have heterogeneous CPUs, we have to manage the list per-TRBE instance to be able to apply the work around as needed. Thus we will need to check if individual CPUs are affected by the erratum. We rely on the arm64 errata framework for the actual description and the discovery of a given erratum, to keep the Erratum work around at a central place and benefit from the code and the advertisement from the kernel. Though we could reuse the "this_cpu_has_cap()" to apply an erratum work around, it is a bit of a heavy operation, as it must go through the "erratum" detection check on the CPU every time it is called (e.g, scanning through a table of affected MIDRs). Since we need to do this check for every session, may be multiple times (depending on the wrok around), we could save the cycles by caching the affected errata per-CPU instance in the per-CPU struct trbe_cpudata. Since we are only interested in the errata affecting the TRBE driver, we only need to track a very few of them per-CPU. Thus we use a local mapping of the CPUCAP for the erratum to avoid bloating up a bitmap for trbe_cpudata. i.e, each arm64 TRBE erratum bit is assigned a "index" within the driver to track. Each trbe instance updates the list of affected erratum at probe time on the CPU. This makes sure that we can easily access the list of errata on a given TRBE instance without much overhead. Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Cc: Anshuman Khandual Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-10-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 69 +++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index cdbb134892f6..fdd60ff4fb3d 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -16,6 +16,8 @@ #define pr_fmt(fmt) DRVNAME ": " fmt #include +#include + #include "coresight-self-hosted-trace.h" #include "coresight-trbe.h" @@ -67,6 +69,34 @@ struct trbe_buf { struct trbe_cpudata *cpudata; }; +/* + * TRBE erratum list + * + * The errata are defined in arm64 generic cpu_errata framework. + * Since the errata work arounds could be applied individually + * to the affected CPUs inside the TRBE driver, we need to know if + * a given CPU is affected by the erratum. Unlike the other erratum + * work arounds, TRBE driver needs to check multiple times during + * a trace session. Thus we need a quicker access to per-CPU + * errata and not issue costly this_cpu_has_cap() everytime. + * We keep a set of the affected errata in trbe_cpudata, per TRBE. + * + * We rely on the corresponding cpucaps to be defined for a given + * TRBE erratum. We map the given cpucap into a TRBE internal number + * to make the tracking of the errata lean. + * + * This helps in : + * - Not duplicating the detection logic + * - Streamlined detection of erratum across the system + */ + +static int trbe_errata_cpucaps[] = { + -1, /* Sentinel, must be the last entry */ +}; + +/* The total number of listed errata in trbe_errata_cpucaps */ +#define TRBE_ERRATA_MAX (ARRAY_SIZE(trbe_errata_cpucaps) - 1) + /* * struct trbe_cpudata: TRBE instance specific data * @trbe_flag - TRBE dirty/access flag support @@ -75,6 +105,7 @@ struct trbe_buf { * @cpu - CPU this TRBE belongs to. * @mode - Mode of current operation. (perf/disabled) * @drvdata - TRBE specific drvdata + * @errata - Bit map for the errata on this TRBE. */ struct trbe_cpudata { bool trbe_flag; @@ -84,6 +115,7 @@ struct trbe_cpudata { enum cs_mode mode; struct trbe_buf *buf; struct trbe_drvdata *drvdata; + DECLARE_BITMAP(errata, TRBE_ERRATA_MAX); }; struct trbe_drvdata { @@ -96,6 +128,25 @@ struct trbe_drvdata { struct platform_device *pdev; }; +static void trbe_check_errata(struct trbe_cpudata *cpudata) +{ + int i; + + for (i = 0; i < TRBE_ERRATA_MAX; i++) { + int cap = trbe_errata_cpucaps[i]; + + if (WARN_ON_ONCE(cap < 0)) + return; + if (this_cpu_has_cap(cap)) + set_bit(i, cpudata->errata); + } +} + +static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i) +{ + return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata); +} + static int trbe_alloc_node(struct perf_event *event) { if (event->cpu == -1) @@ -956,6 +1007,9 @@ cpu_clear: cpumask_clear_cpu(cpu, &drvdata->supported_cpus); } +/* + * Must be called with preemption disabled, for trbe_check_errata(). + */ static void arm_trbe_probe_cpu(void *info) { struct trbe_drvdata *drvdata = info; @@ -983,6 +1037,12 @@ static void arm_trbe_probe_cpu(void *info) goto cpu_clear; } + /* + * Run the TRBE erratum checks, now that we know + * this instance is about to be registered. + */ + trbe_check_errata(cpudata); + cpudata->trbe_align = cpudata->trbe_hw_align; cpudata->trbe_flag = get_trbe_flag_update(trbidr); cpudata->cpu = cpu; @@ -1038,6 +1098,13 @@ static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata) return 0; } +static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata) +{ + preempt_disable(); + arm_trbe_probe_cpu(drvdata); + preempt_enable(); +} + static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node) { struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node); @@ -1049,7 +1116,7 @@ static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node) * initialize it now. */ if (!coresight_get_percpu_sink(cpu)) { - arm_trbe_probe_cpu(drvdata); + arm_trbe_probe_hotplugged_cpu(drvdata); if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) arm_trbe_register_coresight_cpu(drvdata, cpu); if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) From 5cb75f18800bac3c4bfc46f28d3d42d87439d9e2 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:48 +0100 Subject: [PATCH 34/39] coresight: trbe: Workaround TRBE errata overwrite in FILL mode ARM Neoverse-N2 (#2139208) and Cortex-A710(##2119858) suffers from an erratum, which when triggered, might cause the TRBE to overwrite the trace data already collected in FILL mode, in the event of a WRAP. i.e, the TRBE doesn't stop writing the data, instead wraps to the base and could write upto 3 cache line size worth trace. Thus, this could corrupt the trace at the "BASE" pointer. The workaround is to program the write pointer 256bytes from the base, such that if the erratum is triggered, it doesn't overwrite the trace data that was captured. This skipped region could be padded with ignore packets at the end of the session, so that the decoder sees a continuous buffer with some padding at the beginning. The trace data written at the base is considered lost as the limit could have been in the middle of the perf ring buffer, and jumping to the "base" is not acceptable. We set the flags already to indicate that some amount of trace was lost during the FILL event IRQ. So this is fine. One important change with the work around is, we program the TRBBASER_EL1 to current page where we are allowed to write. Otherwise, it could overwrite a region that may be consumed by the perf. Towards this, we always make sure that the "handle->head" and thus the trbe_write is PAGE_SIZE aligned, so that we can set the BASE to the PAGE base and move the TRBPTR to the 256bytes offset. Cc: Mike Leach Cc: Mathieu Poirier Cc: Anshuman Khandual Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-11-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 168 +++++++++++++++++-- 1 file changed, 157 insertions(+), 11 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index fdd60ff4fb3d..2d39e5ecb72c 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -89,14 +89,22 @@ struct trbe_buf { * - Not duplicating the detection logic * - Streamlined detection of erratum across the system */ +#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE 0 static int trbe_errata_cpucaps[] = { + [TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE, -1, /* Sentinel, must be the last entry */ }; /* The total number of listed errata in trbe_errata_cpucaps */ #define TRBE_ERRATA_MAX (ARRAY_SIZE(trbe_errata_cpucaps) - 1) +/* + * Safe limit for the number of bytes that may be overwritten + * when ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE is triggered. + */ +#define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES 256 + /* * struct trbe_cpudata: TRBE instance specific data * @trbe_flag - TRBE dirty/access flag support @@ -147,6 +155,11 @@ static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i) return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata); } +static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata) +{ + return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE); +} + static int trbe_alloc_node(struct perf_event *event) { if (event->cpu == -1) @@ -550,10 +563,13 @@ static void trbe_enable_hw(struct trbe_buf *buf) set_trbe_limit_pointer_enabled(buf->trbe_limit); } -static enum trbe_fault_action trbe_get_fault_act(u64 trbsr) +static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle, + u64 trbsr) { int ec = get_trbe_ec(trbsr); int bsc = get_trbe_bsc(trbsr); + struct trbe_buf *buf = etm_perf_sink_config(handle); + struct trbe_cpudata *cpudata = buf->cpudata; WARN_ON(is_trbe_running(trbsr)); if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr)) @@ -562,10 +578,16 @@ static enum trbe_fault_action trbe_get_fault_act(u64 trbsr) if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT)) return TRBE_FAULT_ACT_FATAL; - if (is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) { - if (get_trbe_write_pointer() == get_trbe_base_pointer()) - return TRBE_FAULT_ACT_WRAP; - } + /* + * If the trbe is affected by TRBE_WORKAROUND_OVERWRITE_FILL_MODE, + * it might write data after a WRAP event in the fill mode. + * Thus the check TRBPTR == TRBBASER will not be honored. + */ + if ((is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) && + (trbe_may_overwrite_in_fill_mode(cpudata) || + get_trbe_write_pointer() == get_trbe_base_pointer())) + return TRBE_FAULT_ACT_WRAP; + return TRBE_FAULT_ACT_SPURIOUS; } @@ -574,6 +596,8 @@ static unsigned long trbe_get_trace_size(struct perf_output_handle *handle, { u64 write; u64 start_off, end_off; + u64 size; + u64 overwrite_skip = TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES; /* * If the TRBE has wrapped around the write pointer has @@ -594,7 +618,18 @@ static unsigned long trbe_get_trace_size(struct perf_output_handle *handle, if (WARN_ON_ONCE(end_off < start_off)) return 0; - return (end_off - start_off); + + size = end_off - start_off; + /* + * If the TRBE is affected by the following erratum, we must fill + * the space we skipped with IGNORE packets. And we are always + * guaranteed to have at least a PAGE_SIZE space in the buffer. + */ + if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) && + !WARN_ON(size < overwrite_skip)) + __trbe_pad_buf(buf, start_off, overwrite_skip); + + return size; } static void *arm_trbe_alloc_buffer(struct coresight_device *csdev, @@ -713,7 +748,7 @@ static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev, clr_trbe_irq(); isb(); - act = trbe_get_fault_act(status); + act = trbe_get_fault_act(handle, status); /* * If this was not due to a WRAP event, we have some * errors and as such buffer is empty. @@ -737,21 +772,117 @@ done: return size; } + +static int trbe_apply_work_around_before_enable(struct trbe_buf *buf) +{ + /* + * TRBE_WORKAROUND_OVERWRITE_FILL_MODE causes the TRBE to overwrite a few cache + * line size from the "TRBBASER_EL1" in the event of a "FILL". + * Thus, we could loose some amount of the trace at the base. + * + * Before Fix: + * + * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT) + * | \/ / + * ------------------------------------------------------------- + * | Pg0 | Pg1 | | | PgN | + * ------------------------------------------------------------- + * + * In the normal course of action, we would set the TRBBASER to the + * beginning of the ring-buffer (normal-BASE). But with the erratum, + * the TRBE could overwrite the contents at the "normal-BASE", after + * hitting the "normal-LIMIT", since it doesn't stop as expected. And + * this is wrong. This could result in overwriting trace collected in + * one of the previous runs, being consumed by the user. So we must + * always make sure that the TRBBASER is within the region + * [head, head+size]. Note that TRBBASER must be PAGE aligned, + * + * After moving the BASE: + * + * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT) + * | \/ / + * ------------------------------------------------------------- + * | | |xyzdef. |.. tuvw| | + * ------------------------------------------------------------- + * / + * New-BASER + * + * Also, we would set the TRBPTR to head (after adjusting for + * alignment) at normal-PTR. This would mean that the last few bytes + * of the trace (say, "xyz") might overwrite the first few bytes of + * trace written ("abc"). More importantly they will appear in what + * userspace sees as the beginning of the trace, which is wrong. We may + * not always have space to move the latest trace "xyz" to the correct + * order as it must appear beyond the LIMIT. (i.e, [head..head+size]). + * Thus it is easier to ignore those bytes than to complicate the + * driver to move it, assuming that the erratum was triggered and + * doing additional checks to see if there is indeed allowed space at + * TRBLIMITR.LIMIT. + * + * Thus the full workaround will move the BASE and the PTR and would + * look like (after padding at the skipped bytes at the end of + * session) : + * + * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT) + * | \/ / + * ------------------------------------------------------------- + * | | |///abc.. |.. rst| | + * ------------------------------------------------------------- + * / | + * New-BASER New-TRBPTR + * + * To summarize, with the work around: + * + * - We always align the offset for the next session to PAGE_SIZE + * (This is to ensure we can program the TRBBASER to this offset + * within the region [head...head+size]). + * + * - At TRBE enable: + * - Set the TRBBASER to the page aligned offset of the current + * proposed write offset. (which is guaranteed to be aligned + * as above) + * - Move the TRBPTR to skip first 256bytes (that might be + * overwritten with the erratum). This ensures that the trace + * generated in the session is not re-written. + * + * - At trace collection: + * - Pad the 256bytes skipped above again with IGNORE packets. + */ + if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) { + if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE))) + return -EINVAL; + buf->trbe_hw_base = buf->trbe_write; + buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES; + } + + return 0; +} + static int __arm_trbe_enable(struct trbe_buf *buf, struct perf_output_handle *handle) { + int ret = 0; + perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW); buf->trbe_limit = compute_trbe_buffer_limit(handle); buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); if (buf->trbe_limit == buf->trbe_base) { - trbe_stop_and_truncate_event(handle); - return -ENOSPC; + ret = -ENOSPC; + goto err; } /* Set the base of the TRBE to the buffer base */ buf->trbe_hw_base = buf->trbe_base; + + ret = trbe_apply_work_around_before_enable(buf); + if (ret) + goto err; + *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle; trbe_enable_hw(buf); return 0; +err: + trbe_stop_and_truncate_event(handle); + return ret; } static int arm_trbe_enable(struct coresight_device *csdev, u32 mode, void *data) @@ -891,7 +1022,7 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) if (!is_perf_trbe(handle)) return IRQ_NONE; - act = trbe_get_fault_act(status); + act = trbe_get_fault_act(handle, status); switch (act) { case TRBE_FAULT_ACT_WRAP: truncated = !!trbe_handle_overflow(handle); @@ -1043,7 +1174,22 @@ static void arm_trbe_probe_cpu(void *info) */ trbe_check_errata(cpudata); - cpudata->trbe_align = cpudata->trbe_hw_align; + /* + * If the TRBE is affected by erratum TRBE_WORKAROUND_OVERWRITE_FILL_MODE, + * we must always program the TBRPTR_EL1, 256bytes from a page + * boundary, with TRBBASER_EL1 set to the page, to prevent + * TRBE over-writing 256bytes at TRBBASER_EL1 on FILL event. + * + * Thus make sure we always align our write pointer to a PAGE_SIZE, + * which also guarantees that we have at least a PAGE_SIZE space in + * the buffer (TRBLIMITR is PAGE aligned) and thus we can skip + * the required bytes at the base. + */ + if (trbe_may_overwrite_in_fill_mode(cpudata)) + cpudata->trbe_align = PAGE_SIZE; + else + cpudata->trbe_align = cpudata->trbe_hw_align; + cpudata->trbe_flag = get_trbe_flag_update(trbidr); cpudata->cpu = cpu; cpudata->drvdata = drvdata; From 7c2cc5e26cc0c6bc2478ac203e3fe71950418ea0 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:49 +0100 Subject: [PATCH 35/39] coresight: trbe: Add a helper to determine the minimum buffer size For the TRBE to operate, we need a minimum space available to collect meaningful trace session. This is currently a few bytes, but we may need to extend this for working around errata. So, abstract this into a helper function. Cc: Anshuman Khandual Cc: Mike Leach Cc: Mathieu Poirier Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-12-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 2d39e5ecb72c..02a639ece2af 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -303,6 +303,11 @@ static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle) return buf->nr_pages * PAGE_SIZE; } +static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle) +{ + return TRBE_TRACE_MIN_BUF_SIZE; +} + /* * TRBE Limit Calculation * @@ -473,7 +478,7 @@ static unsigned long trbe_normal_offset(struct perf_output_handle *handle) * have space for a meaningful run, we rather pad it * and start fresh. */ - if (limit && (limit - head < TRBE_TRACE_MIN_BUF_SIZE)) { + if (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) { trbe_pad_buf(handle, limit - head); limit = __trbe_normal_offset(handle); } From adf35d058617c94810c03628efd7ae7821f3c5e7 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:50 +0100 Subject: [PATCH 36/39] coresight: trbe: Make sure we have enough space The TRBE driver makes sure that there is enough space for a meaningful run, otherwise pads the given space and restarts the offset calculation once. But there is no guarantee that we may find space or hit "no space". Make sure that we repeat the step until, either : - We have the minimum space OR - There is NO space at all. Cc: Anshuman Khandual Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-13-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 02a639ece2af..11ff33ec83b9 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -477,10 +477,14 @@ static unsigned long trbe_normal_offset(struct perf_output_handle *handle) * If the head is too close to the limit and we don't * have space for a meaningful run, we rather pad it * and start fresh. + * + * We might have to do this more than once to make sure + * we have enough required space. */ - if (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) { + while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) { trbe_pad_buf(handle, limit - head); limit = __trbe_normal_offset(handle); + head = PERF_IDX2OFF(handle->head, buf); } return limit; } From f9efc79d0ab9b780bf74c624616bd50c6cefdeeb Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:51 +0100 Subject: [PATCH 37/39] coresight: trbe: Work around write to out of range TRBE implementations affected by Arm erratum (2253138 or 2224489), could write to the next address after the TRBLIMITR.LIMIT, instead of wrapping to the TRBBASER. This implies that the TRBE could potentially corrupt : - A page used by the rest of the kernel/user (if the LIMIT = end of perf ring buffer) - A page within the ring buffer, but outside the driver's range. [head, head + size]. This may contain some trace data, may be consumed by the userspace. We workaround this erratum by : - Making sure that there is at least an extra PAGE space left in the TRBE's range than we normally assign. This will be additional to other restrictions (e.g, the TRBE alignment for working around TRBE_WORKAROUND_OVERWRITE_IN_FILL_MODE, where there is a minimum of PAGE_SIZE. Thus we would have 2 * PAGE_SIZE) - Adjust the LIMIT to leave the last PAGE_SIZE out of the TRBE's allowed range (i.e, TRBEBASER...TRBLIMITR.LIMIT), by : TRBLIMITR.LIMIT -= PAGE_SIZE Cc: Anshuman Khandual Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Reviewed-by: Anshuman Khandual Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-14-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- drivers/hwtracing/coresight/coresight-trbe.c | 63 +++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 11ff33ec83b9..276862c07e32 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -90,9 +90,11 @@ struct trbe_buf { * - Streamlined detection of erratum across the system */ #define TRBE_WORKAROUND_OVERWRITE_FILL_MODE 0 +#define TRBE_WORKAROUND_WRITE_OUT_OF_RANGE 1 static int trbe_errata_cpucaps[] = { [TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE, + [TRBE_WORKAROUND_WRITE_OUT_OF_RANGE] = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE, -1, /* Sentinel, must be the last entry */ }; @@ -160,6 +162,11 @@ static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata) return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE); } +static inline bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata) +{ + return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE); +} + static int trbe_alloc_node(struct perf_event *event) { if (event->cpu == -1) @@ -305,7 +312,21 @@ static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle) static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle) { - return TRBE_TRACE_MIN_BUF_SIZE; + u64 size = TRBE_TRACE_MIN_BUF_SIZE; + struct trbe_buf *buf = etm_perf_sink_config(handle); + struct trbe_cpudata *cpudata = buf->cpudata; + + /* + * When the TRBE is affected by an erratum that could make it + * write to the next "virtually addressed" page beyond the LIMIT. + * We need to make sure there is always a PAGE after the LIMIT, + * within the buffer. Thus we ensure there is at least an extra + * page than normal. With this we could then adjust the LIMIT + * pointer down by a PAGE later. + */ + if (trbe_may_write_out_of_range(cpudata)) + size += PAGE_SIZE; + return size; } /* @@ -611,6 +632,17 @@ static unsigned long trbe_get_trace_size(struct perf_output_handle *handle, /* * If the TRBE has wrapped around the write pointer has * wrapped and should be treated as limit. + * + * When the TRBE is affected by TRBE_WORKAROUND_WRITE_OUT_OF_RANGE, + * it may write upto 64bytes beyond the "LIMIT". The driver already + * keeps a valid page next to the LIMIT and we could potentially + * consume the trace data that may have been collected there. But we + * cannot be really sure it is available, and the TRBPTR may not + * indicate the same. Also, affected cores are also affected by another + * erratum which forces the PAGE_SIZE alignment on the TRBPTR, and thus + * could potentially pad an entire PAGE_SIZE - 64bytes, to get those + * 64bytes. Thus we ignore the potential triggering of the erratum + * on WRAP and limit the data to LIMIT. */ if (wrap) write = get_trbe_limit_pointer(); @@ -864,6 +896,35 @@ static int trbe_apply_work_around_before_enable(struct trbe_buf *buf) buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES; } + /* + * TRBE_WORKAROUND_WRITE_OUT_OF_RANGE could cause the TRBE to write to + * the next page after the TRBLIMITR.LIMIT. For perf, the "next page" + * may be: + * - The page beyond the ring buffer. This could mean, TRBE could + * corrupt another entity (kernel / user) + * - A portion of the "ring buffer" consumed by the userspace. + * i.e, a page outisde [head, head + size]. + * + * We work around this by: + * - Making sure that we have at least an extra space of PAGE left + * in the ring buffer [head, head + size], than we normally do + * without the erratum. See trbe_min_trace_buf_size(). + * + * - Adjust the TRBLIMITR.LIMIT to leave the extra PAGE outside + * the TRBE's range (i.e [TRBBASER, TRBLIMITR.LIMI] ). + */ + if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) { + s64 space = buf->trbe_limit - buf->trbe_write; + /* + * We must have more than a PAGE_SIZE worth space in the proposed + * range for the TRBE. + */ + if (WARN_ON(space <= PAGE_SIZE || + !IS_ALIGNED(buf->trbe_limit, PAGE_SIZE))) + return -EINVAL; + buf->trbe_limit -= PAGE_SIZE; + } + return 0; } From 74b2740f57cc161297b800ecb0e1e549d13c3bb9 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:52 +0100 Subject: [PATCH 38/39] arm64: errata: Enable workaround for TRBE overwrite in FILL mode With the workaround enabled in TRBE, enable the config entries to be built without COMPILE_TEST Cc: Catalin Marinas Cc: Will Deacon Reviewed-by: Anshuman Khandual Acked-by: Will Deacon Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-15-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- arch/arm64/Kconfig | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index a500af15ebd5..d44b4442611e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -672,7 +672,6 @@ config ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE config ARM64_ERRATUM_2119858 bool "Cortex-A710: 2119858: workaround TRBE overwriting trace data in FILL mode" default y - depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in depends on CORESIGHT_TRBE select ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE help @@ -691,7 +690,6 @@ config ARM64_ERRATUM_2119858 config ARM64_ERRATUM_2139208 bool "Neoverse-N2: 2139208: workaround TRBE overwriting trace data in FILL mode" default y - depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in depends on CORESIGHT_TRBE select ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE help From 561ced0bb90a4be298b7db5fb54f29731d74a3f6 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Tue, 19 Oct 2021 17:31:53 +0100 Subject: [PATCH 39/39] arm64: errata: Enable TRBE workaround for write to out-of-range address With the TRBE driver workaround available, enable the config symbols to be built without COMPILE_TEST Cc: Catalin Marinas Cc: Will Deacon Reviewed-by: Anshuman Khandual Acked-by: Will Deacon Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20211019163153.3692640-16-suzuki.poulose@arm.com Signed-off-by: Mathieu Poirier --- arch/arm64/Kconfig | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index d44b4442611e..9c33ce54509a 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -743,7 +743,6 @@ config ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE config ARM64_ERRATUM_2253138 bool "Neoverse-N2: 2253138: workaround TRBE writing to address out-of-range" - depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in depends on CORESIGHT_TRBE default y select ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE @@ -762,7 +761,6 @@ config ARM64_ERRATUM_2253138 config ARM64_ERRATUM_2224489 bool "Cortex-A710: 2224489: workaround TRBE writing to address out-of-range" - depends on COMPILE_TEST # Until the CoreSight TRBE driver changes are in depends on CORESIGHT_TRBE default y select ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE