1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * preemptoff and irqoff tracepoints |
4 | * |
5 | * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org> |
6 | */ |
7 | |
8 | #include <linux/kallsyms.h> |
9 | #include <linux/uaccess.h> |
10 | #include <linux/module.h> |
11 | #include <linux/ftrace.h> |
12 | #include <linux/kprobes.h> |
13 | #include "trace.h" |
14 | |
15 | #define CREATE_TRACE_POINTS |
16 | #include <trace/events/preemptirq.h> |
17 | |
18 | /* |
19 | * Use regular trace points on architectures that implement noinstr |
20 | * tooling: these calls will only happen with RCU enabled, which can |
21 | * use a regular tracepoint. |
22 | * |
23 | * On older architectures, use the rcuidle tracing methods (which |
24 | * aren't NMI-safe - so exclude NMI contexts): |
25 | */ |
26 | #ifdef CONFIG_ARCH_WANTS_NO_INSTR |
27 | #define trace(point) trace_##point |
28 | #else |
29 | #define trace(point) if (!in_nmi()) trace_##point##_rcuidle |
30 | #endif |
31 | |
32 | #ifdef CONFIG_TRACE_IRQFLAGS |
33 | /* Per-cpu variable to prevent redundant calls when IRQs already off */ |
34 | static DEFINE_PER_CPU(int, tracing_irq_cpu); |
35 | |
36 | /* |
37 | * Like trace_hardirqs_on() but without the lockdep invocation. This is |
38 | * used in the low level entry code where the ordering vs. RCU is important |
39 | * and lockdep uses a staged approach which splits the lockdep hardirq |
40 | * tracking into a RCU on and a RCU off section. |
41 | */ |
42 | void trace_hardirqs_on_prepare(void) |
43 | { |
44 | if (this_cpu_read(tracing_irq_cpu)) { |
45 | trace(irq_enable)(CALLER_ADDR0, CALLER_ADDR1); |
46 | tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); |
47 | this_cpu_write(tracing_irq_cpu, 0); |
48 | } |
49 | } |
50 | EXPORT_SYMBOL(trace_hardirqs_on_prepare); |
51 | NOKPROBE_SYMBOL(trace_hardirqs_on_prepare); |
52 | |
53 | void trace_hardirqs_on(void) |
54 | { |
55 | if (this_cpu_read(tracing_irq_cpu)) { |
56 | trace(irq_enable)(CALLER_ADDR0, CALLER_ADDR1); |
57 | tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); |
58 | this_cpu_write(tracing_irq_cpu, 0); |
59 | } |
60 | |
61 | lockdep_hardirqs_on_prepare(); |
62 | lockdep_hardirqs_on(CALLER_ADDR0); |
63 | } |
64 | EXPORT_SYMBOL(trace_hardirqs_on); |
65 | NOKPROBE_SYMBOL(trace_hardirqs_on); |
66 | |
67 | /* |
68 | * Like trace_hardirqs_off() but without the lockdep invocation. This is |
69 | * used in the low level entry code where the ordering vs. RCU is important |
70 | * and lockdep uses a staged approach which splits the lockdep hardirq |
71 | * tracking into a RCU on and a RCU off section. |
72 | */ |
73 | void trace_hardirqs_off_finish(void) |
74 | { |
75 | if (!this_cpu_read(tracing_irq_cpu)) { |
76 | this_cpu_write(tracing_irq_cpu, 1); |
77 | tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); |
78 | trace(irq_disable)(CALLER_ADDR0, CALLER_ADDR1); |
79 | } |
80 | |
81 | } |
82 | EXPORT_SYMBOL(trace_hardirqs_off_finish); |
83 | NOKPROBE_SYMBOL(trace_hardirqs_off_finish); |
84 | |
85 | void trace_hardirqs_off(void) |
86 | { |
87 | lockdep_hardirqs_off(CALLER_ADDR0); |
88 | |
89 | if (!this_cpu_read(tracing_irq_cpu)) { |
90 | this_cpu_write(tracing_irq_cpu, 1); |
91 | tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); |
92 | trace(irq_disable)(CALLER_ADDR0, CALLER_ADDR1); |
93 | } |
94 | } |
95 | EXPORT_SYMBOL(trace_hardirqs_off); |
96 | NOKPROBE_SYMBOL(trace_hardirqs_off); |
97 | #endif /* CONFIG_TRACE_IRQFLAGS */ |
98 | |
99 | #ifdef CONFIG_TRACE_PREEMPT_TOGGLE |
100 | |
101 | void trace_preempt_on(unsigned long a0, unsigned long a1) |
102 | { |
103 | trace(preempt_enable)(ip: a0, parent_ip: a1); |
104 | tracer_preempt_on(a0, a1); |
105 | } |
106 | |
107 | void trace_preempt_off(unsigned long a0, unsigned long a1) |
108 | { |
109 | trace(preempt_disable)(ip: a0, parent_ip: a1); |
110 | tracer_preempt_off(a0, a1); |
111 | } |
112 | #endif |
113 | |