1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #undef TRACE_SYSTEM |
3 | #define TRACE_SYSTEM rcu |
4 | |
5 | #if !defined(_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ) |
6 | #define _TRACE_RCU_H |
7 | |
8 | #include <linux/tracepoint.h> |
9 | |
10 | #ifdef CONFIG_RCU_TRACE |
11 | #define TRACE_EVENT_RCU TRACE_EVENT |
12 | #else |
13 | #define TRACE_EVENT_RCU TRACE_EVENT_NOP |
14 | #endif |
15 | |
16 | /* |
17 | * Tracepoint for start/end markers used for utilization calculations. |
18 | * By convention, the string is of the following forms: |
19 | * |
20 | * "Start <activity>" -- Mark the start of the specified activity, |
21 | * such as "context switch". Nesting is permitted. |
22 | * "End <activity>" -- Mark the end of the specified activity. |
23 | * |
24 | * An "@" character within "<activity>" is a comment character: Data |
25 | * reduction scripts will ignore the "@" and the remainder of the line. |
26 | */ |
27 | TRACE_EVENT(rcu_utilization, |
28 | |
29 | TP_PROTO(const char *s), |
30 | |
31 | TP_ARGS(s), |
32 | |
33 | TP_STRUCT__entry( |
34 | __field(const char *, s) |
35 | ), |
36 | |
37 | TP_fast_assign( |
38 | __entry->s = s; |
39 | ), |
40 | |
41 | TP_printk("%s" , __entry->s) |
42 | ); |
43 | |
44 | #if defined(CONFIG_TREE_RCU) |
45 | |
46 | /* |
47 | * Tracepoint for grace-period events. Takes a string identifying the |
48 | * RCU flavor, the grace-period number, and a string identifying the |
49 | * grace-period-related event as follows: |
50 | * |
51 | * "AccReadyCB": CPU accelerates new callbacks to RCU_NEXT_READY_TAIL. |
52 | * "AccWaitCB": CPU accelerates new callbacks to RCU_WAIT_TAIL. |
53 | * "newreq": Request a new grace period. |
54 | * "start": Start a grace period. |
55 | * "cpustart": CPU first notices a grace-period start. |
56 | * "cpuqs": CPU passes through a quiescent state. |
57 | * "cpuonl": CPU comes online. |
58 | * "cpuofl": CPU goes offline. |
59 | * "cpuofl-bgp": CPU goes offline while blocking a grace period. |
60 | * "reqwait": GP kthread sleeps waiting for grace-period request. |
61 | * "reqwaitsig": GP kthread awakened by signal from reqwait state. |
62 | * "fqswait": GP kthread waiting until time to force quiescent states. |
63 | * "fqsstart": GP kthread starts forcing quiescent states. |
64 | * "fqsend": GP kthread done forcing quiescent states. |
65 | * "fqswaitsig": GP kthread awakened by signal from fqswait state. |
66 | * "end": End a grace period. |
67 | * "cpuend": CPU first notices a grace-period end. |
68 | */ |
69 | TRACE_EVENT_RCU(rcu_grace_period, |
70 | |
71 | TP_PROTO(const char *rcuname, unsigned long gp_seq, const char *gpevent), |
72 | |
73 | TP_ARGS(rcuname, gp_seq, gpevent), |
74 | |
75 | TP_STRUCT__entry( |
76 | __field(const char *, rcuname) |
77 | __field(long, gp_seq) |
78 | __field(const char *, gpevent) |
79 | ), |
80 | |
81 | TP_fast_assign( |
82 | __entry->rcuname = rcuname; |
83 | __entry->gp_seq = (long)gp_seq; |
84 | __entry->gpevent = gpevent; |
85 | ), |
86 | |
87 | TP_printk("%s %ld %s" , |
88 | __entry->rcuname, __entry->gp_seq, __entry->gpevent) |
89 | ); |
90 | |
91 | /* |
92 | * Tracepoint for future grace-period events. The caller should pull |
93 | * the data from the rcu_node structure, other than rcuname, which comes |
94 | * from the rcu_state structure, and event, which is one of the following: |
95 | * |
96 | * "Cleanup": Clean up rcu_node structure after previous GP. |
97 | * "CleanupMore": Clean up, and another GP is needed. |
98 | * "EndWait": Complete wait. |
99 | * "NoGPkthread": The RCU grace-period kthread has not yet started. |
100 | * "Prestarted": Someone beat us to the request |
101 | * "Startedleaf": Leaf node marked for future GP. |
102 | * "Startedleafroot": All nodes from leaf to root marked for future GP. |
103 | * "Startedroot": Requested a nocb grace period based on root-node data. |
104 | * "Startleaf": Request a grace period based on leaf-node data. |
105 | * "StartWait": Start waiting for the requested grace period. |
106 | */ |
107 | TRACE_EVENT_RCU(rcu_future_grace_period, |
108 | |
109 | TP_PROTO(const char *rcuname, unsigned long gp_seq, |
110 | unsigned long gp_seq_req, u8 level, int grplo, int grphi, |
111 | const char *gpevent), |
112 | |
113 | TP_ARGS(rcuname, gp_seq, gp_seq_req, level, grplo, grphi, gpevent), |
114 | |
115 | TP_STRUCT__entry( |
116 | __field(const char *, rcuname) |
117 | __field(long, gp_seq) |
118 | __field(long, gp_seq_req) |
119 | __field(u8, level) |
120 | __field(int, grplo) |
121 | __field(int, grphi) |
122 | __field(const char *, gpevent) |
123 | ), |
124 | |
125 | TP_fast_assign( |
126 | __entry->rcuname = rcuname; |
127 | __entry->gp_seq = (long)gp_seq; |
128 | __entry->gp_seq_req = (long)gp_seq_req; |
129 | __entry->level = level; |
130 | __entry->grplo = grplo; |
131 | __entry->grphi = grphi; |
132 | __entry->gpevent = gpevent; |
133 | ), |
134 | |
135 | TP_printk("%s %ld %ld %u %d %d %s" , |
136 | __entry->rcuname, (long)__entry->gp_seq, (long)__entry->gp_seq_req, __entry->level, |
137 | __entry->grplo, __entry->grphi, __entry->gpevent) |
138 | ); |
139 | |
140 | /* |
141 | * Tracepoint for grace-period-initialization events. These are |
142 | * distinguished by the type of RCU, the new grace-period number, the |
143 | * rcu_node structure level, the starting and ending CPU covered by the |
144 | * rcu_node structure, and the mask of CPUs that will be waited for. |
145 | * All but the type of RCU are extracted from the rcu_node structure. |
146 | */ |
147 | TRACE_EVENT_RCU(rcu_grace_period_init, |
148 | |
149 | TP_PROTO(const char *rcuname, unsigned long gp_seq, u8 level, |
150 | int grplo, int grphi, unsigned long qsmask), |
151 | |
152 | TP_ARGS(rcuname, gp_seq, level, grplo, grphi, qsmask), |
153 | |
154 | TP_STRUCT__entry( |
155 | __field(const char *, rcuname) |
156 | __field(long, gp_seq) |
157 | __field(u8, level) |
158 | __field(int, grplo) |
159 | __field(int, grphi) |
160 | __field(unsigned long, qsmask) |
161 | ), |
162 | |
163 | TP_fast_assign( |
164 | __entry->rcuname = rcuname; |
165 | __entry->gp_seq = (long)gp_seq; |
166 | __entry->level = level; |
167 | __entry->grplo = grplo; |
168 | __entry->grphi = grphi; |
169 | __entry->qsmask = qsmask; |
170 | ), |
171 | |
172 | TP_printk("%s %ld %u %d %d %lx" , |
173 | __entry->rcuname, __entry->gp_seq, __entry->level, |
174 | __entry->grplo, __entry->grphi, __entry->qsmask) |
175 | ); |
176 | |
177 | /* |
178 | * Tracepoint for expedited grace-period events. Takes a string identifying |
179 | * the RCU flavor, the expedited grace-period sequence number, and a string |
180 | * identifying the grace-period-related event as follows: |
181 | * |
182 | * "snap": Captured snapshot of expedited grace period sequence number. |
183 | * "start": Started a real expedited grace period. |
184 | * "reset": Started resetting the tree |
185 | * "select": Started selecting the CPUs to wait on. |
186 | * "selectofl": Selected CPU partially offline. |
187 | * "startwait": Started waiting on selected CPUs. |
188 | * "end": Ended a real expedited grace period. |
189 | * "endwake": Woke piggybackers up. |
190 | * "done": Someone else did the expedited grace period for us. |
191 | */ |
192 | TRACE_EVENT_RCU(rcu_exp_grace_period, |
193 | |
194 | TP_PROTO(const char *rcuname, unsigned long gpseq, const char *gpevent), |
195 | |
196 | TP_ARGS(rcuname, gpseq, gpevent), |
197 | |
198 | TP_STRUCT__entry( |
199 | __field(const char *, rcuname) |
200 | __field(long, gpseq) |
201 | __field(const char *, gpevent) |
202 | ), |
203 | |
204 | TP_fast_assign( |
205 | __entry->rcuname = rcuname; |
206 | __entry->gpseq = (long)gpseq; |
207 | __entry->gpevent = gpevent; |
208 | ), |
209 | |
210 | TP_printk("%s %ld %s" , |
211 | __entry->rcuname, __entry->gpseq, __entry->gpevent) |
212 | ); |
213 | |
214 | /* |
215 | * Tracepoint for expedited grace-period funnel-locking events. Takes a |
216 | * string identifying the RCU flavor, an integer identifying the rcu_node |
217 | * combining-tree level, another pair of integers identifying the lowest- |
218 | * and highest-numbered CPU associated with the current rcu_node structure, |
219 | * and a string. identifying the grace-period-related event as follows: |
220 | * |
221 | * "nxtlvl": Advance to next level of rcu_node funnel |
222 | * "wait": Wait for someone else to do expedited GP |
223 | */ |
224 | TRACE_EVENT_RCU(rcu_exp_funnel_lock, |
225 | |
226 | TP_PROTO(const char *rcuname, u8 level, int grplo, int grphi, |
227 | const char *gpevent), |
228 | |
229 | TP_ARGS(rcuname, level, grplo, grphi, gpevent), |
230 | |
231 | TP_STRUCT__entry( |
232 | __field(const char *, rcuname) |
233 | __field(u8, level) |
234 | __field(int, grplo) |
235 | __field(int, grphi) |
236 | __field(const char *, gpevent) |
237 | ), |
238 | |
239 | TP_fast_assign( |
240 | __entry->rcuname = rcuname; |
241 | __entry->level = level; |
242 | __entry->grplo = grplo; |
243 | __entry->grphi = grphi; |
244 | __entry->gpevent = gpevent; |
245 | ), |
246 | |
247 | TP_printk("%s %d %d %d %s" , |
248 | __entry->rcuname, __entry->level, __entry->grplo, |
249 | __entry->grphi, __entry->gpevent) |
250 | ); |
251 | |
252 | #ifdef CONFIG_RCU_NOCB_CPU |
253 | /* |
254 | * Tracepoint for RCU no-CBs CPU callback handoffs. This event is intended |
255 | * to assist debugging of these handoffs. |
256 | * |
257 | * The first argument is the name of the RCU flavor, and the second is |
258 | * the number of the offloaded CPU are extracted. The third and final |
259 | * argument is a string as follows: |
260 | * |
261 | * "AlreadyAwake": The to-be-awakened rcuo kthread is already awake. |
262 | * "Bypass": rcuo GP kthread sees non-empty ->nocb_bypass. |
263 | * "CBSleep": rcuo CB kthread sleeping waiting for CBs. |
264 | * "Check": rcuo GP kthread checking specified CPU for work. |
265 | * "DeferredWake": Timer expired or polled check, time to wake. |
266 | * "DoWake": The to-be-awakened rcuo kthread needs to be awakened. |
267 | * "EndSleep": Done waiting for GP for !rcu_nocb_poll. |
268 | * "FirstBQ": New CB to empty ->nocb_bypass (->cblist maybe non-empty). |
269 | * "FirstBQnoWake": FirstBQ plus rcuo kthread need not be awakened. |
270 | * "FirstBQwake": FirstBQ plus rcuo kthread must be awakened. |
271 | * "FirstQ": New CB to empty ->cblist (->nocb_bypass maybe non-empty). |
272 | * "NeedWaitGP": rcuo GP kthread must wait on a grace period. |
273 | * "Poll": Start of new polling cycle for rcu_nocb_poll. |
274 | * "Sleep": Sleep waiting for GP for !rcu_nocb_poll. |
275 | * "Timer": Deferred-wake timer expired. |
276 | * "WakeEmptyIsDeferred": Wake rcuo kthread later, first CB to empty list. |
277 | * "WakeEmpty": Wake rcuo kthread, first CB to empty list. |
278 | * "WakeNot": Don't wake rcuo kthread. |
279 | * "WakeNotPoll": Don't wake rcuo kthread because it is polling. |
280 | * "WakeOvfIsDeferred": Wake rcuo kthread later, CB list is huge. |
281 | * "WakeBypassIsDeferred": Wake rcuo kthread later, bypass list is contended. |
282 | * "WokeEmpty": rcuo CB kthread woke to find empty list. |
283 | */ |
284 | TRACE_EVENT_RCU(rcu_nocb_wake, |
285 | |
286 | TP_PROTO(const char *rcuname, int cpu, const char *reason), |
287 | |
288 | TP_ARGS(rcuname, cpu, reason), |
289 | |
290 | TP_STRUCT__entry( |
291 | __field(const char *, rcuname) |
292 | __field(int, cpu) |
293 | __field(const char *, reason) |
294 | ), |
295 | |
296 | TP_fast_assign( |
297 | __entry->rcuname = rcuname; |
298 | __entry->cpu = cpu; |
299 | __entry->reason = reason; |
300 | ), |
301 | |
302 | TP_printk("%s %d %s" , __entry->rcuname, __entry->cpu, __entry->reason) |
303 | ); |
304 | #endif |
305 | |
306 | /* |
307 | * Tracepoint for tasks blocking within preemptible-RCU read-side |
308 | * critical sections. Track the type of RCU (which one day might |
309 | * include SRCU), the grace-period number that the task is blocking |
310 | * (the current or the next), and the task's PID. |
311 | */ |
312 | TRACE_EVENT_RCU(rcu_preempt_task, |
313 | |
314 | TP_PROTO(const char *rcuname, int pid, unsigned long gp_seq), |
315 | |
316 | TP_ARGS(rcuname, pid, gp_seq), |
317 | |
318 | TP_STRUCT__entry( |
319 | __field(const char *, rcuname) |
320 | __field(long, gp_seq) |
321 | __field(int, pid) |
322 | ), |
323 | |
324 | TP_fast_assign( |
325 | __entry->rcuname = rcuname; |
326 | __entry->gp_seq = (long)gp_seq; |
327 | __entry->pid = pid; |
328 | ), |
329 | |
330 | TP_printk("%s %ld %d" , |
331 | __entry->rcuname, __entry->gp_seq, __entry->pid) |
332 | ); |
333 | |
334 | /* |
335 | * Tracepoint for tasks that blocked within a given preemptible-RCU |
336 | * read-side critical section exiting that critical section. Track the |
337 | * type of RCU (which one day might include SRCU) and the task's PID. |
338 | */ |
339 | TRACE_EVENT_RCU(rcu_unlock_preempted_task, |
340 | |
341 | TP_PROTO(const char *rcuname, unsigned long gp_seq, int pid), |
342 | |
343 | TP_ARGS(rcuname, gp_seq, pid), |
344 | |
345 | TP_STRUCT__entry( |
346 | __field(const char *, rcuname) |
347 | __field(long, gp_seq) |
348 | __field(int, pid) |
349 | ), |
350 | |
351 | TP_fast_assign( |
352 | __entry->rcuname = rcuname; |
353 | __entry->gp_seq = (long)gp_seq; |
354 | __entry->pid = pid; |
355 | ), |
356 | |
357 | TP_printk("%s %ld %d" , __entry->rcuname, __entry->gp_seq, __entry->pid) |
358 | ); |
359 | |
360 | /* |
361 | * Tracepoint for quiescent-state-reporting events. These are |
362 | * distinguished by the type of RCU, the grace-period number, the |
363 | * mask of quiescent lower-level entities, the rcu_node structure level, |
364 | * the starting and ending CPU covered by the rcu_node structure, and |
365 | * whether there are any blocked tasks blocking the current grace period. |
366 | * All but the type of RCU are extracted from the rcu_node structure. |
367 | */ |
368 | TRACE_EVENT_RCU(rcu_quiescent_state_report, |
369 | |
370 | TP_PROTO(const char *rcuname, unsigned long gp_seq, |
371 | unsigned long mask, unsigned long qsmask, |
372 | u8 level, int grplo, int grphi, int gp_tasks), |
373 | |
374 | TP_ARGS(rcuname, gp_seq, mask, qsmask, level, grplo, grphi, gp_tasks), |
375 | |
376 | TP_STRUCT__entry( |
377 | __field(const char *, rcuname) |
378 | __field(long, gp_seq) |
379 | __field(unsigned long, mask) |
380 | __field(unsigned long, qsmask) |
381 | __field(u8, level) |
382 | __field(int, grplo) |
383 | __field(int, grphi) |
384 | __field(u8, gp_tasks) |
385 | ), |
386 | |
387 | TP_fast_assign( |
388 | __entry->rcuname = rcuname; |
389 | __entry->gp_seq = (long)gp_seq; |
390 | __entry->mask = mask; |
391 | __entry->qsmask = qsmask; |
392 | __entry->level = level; |
393 | __entry->grplo = grplo; |
394 | __entry->grphi = grphi; |
395 | __entry->gp_tasks = gp_tasks; |
396 | ), |
397 | |
398 | TP_printk("%s %ld %lx>%lx %u %d %d %u" , |
399 | __entry->rcuname, __entry->gp_seq, |
400 | __entry->mask, __entry->qsmask, __entry->level, |
401 | __entry->grplo, __entry->grphi, __entry->gp_tasks) |
402 | ); |
403 | |
404 | /* |
405 | * Tracepoint for quiescent states detected by force_quiescent_state(). |
406 | * These trace events include the type of RCU, the grace-period number |
407 | * that was blocked by the CPU, the CPU itself, and the type of quiescent |
408 | * state, which can be "dti" for dyntick-idle mode or "kick" when kicking |
409 | * a CPU that has been in dyntick-idle mode for too long. |
410 | */ |
411 | TRACE_EVENT_RCU(rcu_fqs, |
412 | |
413 | TP_PROTO(const char *rcuname, unsigned long gp_seq, int cpu, const char *qsevent), |
414 | |
415 | TP_ARGS(rcuname, gp_seq, cpu, qsevent), |
416 | |
417 | TP_STRUCT__entry( |
418 | __field(const char *, rcuname) |
419 | __field(long, gp_seq) |
420 | __field(int, cpu) |
421 | __field(const char *, qsevent) |
422 | ), |
423 | |
424 | TP_fast_assign( |
425 | __entry->rcuname = rcuname; |
426 | __entry->gp_seq = (long)gp_seq; |
427 | __entry->cpu = cpu; |
428 | __entry->qsevent = qsevent; |
429 | ), |
430 | |
431 | TP_printk("%s %ld %d %s" , |
432 | __entry->rcuname, __entry->gp_seq, |
433 | __entry->cpu, __entry->qsevent) |
434 | ); |
435 | |
436 | /* |
437 | * Tracepoint for RCU stall events. Takes a string identifying the RCU flavor |
438 | * and a string identifying which function detected the RCU stall as follows: |
439 | * |
440 | * "StallDetected": Scheduler-tick detects other CPU's stalls. |
441 | * "SelfDetected": Scheduler-tick detects a current CPU's stall. |
442 | * "ExpeditedStall": Expedited grace period detects stalls. |
443 | */ |
444 | TRACE_EVENT(rcu_stall_warning, |
445 | |
446 | TP_PROTO(const char *rcuname, const char *msg), |
447 | |
448 | TP_ARGS(rcuname, msg), |
449 | |
450 | TP_STRUCT__entry( |
451 | __field(const char *, rcuname) |
452 | __field(const char *, msg) |
453 | ), |
454 | |
455 | TP_fast_assign( |
456 | __entry->rcuname = rcuname; |
457 | __entry->msg = msg; |
458 | ), |
459 | |
460 | TP_printk("%s %s" , |
461 | __entry->rcuname, __entry->msg) |
462 | ); |
463 | |
464 | #endif /* #if defined(CONFIG_TREE_RCU) */ |
465 | |
466 | /* |
467 | * Tracepoint for dyntick-idle entry/exit events. These take 2 strings |
468 | * as argument: |
469 | * polarity: "Start", "End", "StillNonIdle" for entering, exiting or still not |
470 | * being in dyntick-idle mode. |
471 | * context: "USER" or "IDLE" or "IRQ". |
472 | * NMIs nested in IRQs are inferred with dynticks_nesting > 1 in IRQ context. |
473 | * |
474 | * These events also take a pair of numbers, which indicate the nesting |
475 | * depth before and after the event of interest, and a third number that is |
476 | * the ->dynticks counter. Note that task-related and interrupt-related |
477 | * events use two separate counters, and that the "++=" and "--=" events |
478 | * for irq/NMI will change the counter by two, otherwise by one. |
479 | */ |
480 | TRACE_EVENT_RCU(rcu_dyntick, |
481 | |
482 | TP_PROTO(const char *polarity, long oldnesting, long newnesting, int dynticks), |
483 | |
484 | TP_ARGS(polarity, oldnesting, newnesting, dynticks), |
485 | |
486 | TP_STRUCT__entry( |
487 | __field(const char *, polarity) |
488 | __field(long, oldnesting) |
489 | __field(long, newnesting) |
490 | __field(int, dynticks) |
491 | ), |
492 | |
493 | TP_fast_assign( |
494 | __entry->polarity = polarity; |
495 | __entry->oldnesting = oldnesting; |
496 | __entry->newnesting = newnesting; |
497 | __entry->dynticks = dynticks; |
498 | ), |
499 | |
500 | TP_printk("%s %lx %lx %#3x" , __entry->polarity, |
501 | __entry->oldnesting, __entry->newnesting, |
502 | __entry->dynticks & 0xfff) |
503 | ); |
504 | |
505 | /* |
506 | * Tracepoint for the registration of a single RCU callback function. |
507 | * The first argument is the type of RCU, the second argument is |
508 | * a pointer to the RCU callback itself, the third element is the |
509 | * number of lazy callbacks queued, and the fourth element is the |
510 | * total number of callbacks queued. |
511 | */ |
512 | TRACE_EVENT_RCU(rcu_callback, |
513 | |
514 | TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen), |
515 | |
516 | TP_ARGS(rcuname, rhp, qlen), |
517 | |
518 | TP_STRUCT__entry( |
519 | __field(const char *, rcuname) |
520 | __field(void *, rhp) |
521 | __field(void *, func) |
522 | __field(long, qlen) |
523 | ), |
524 | |
525 | TP_fast_assign( |
526 | __entry->rcuname = rcuname; |
527 | __entry->rhp = rhp; |
528 | __entry->func = rhp->func; |
529 | __entry->qlen = qlen; |
530 | ), |
531 | |
532 | TP_printk("%s rhp=%p func=%ps %ld" , |
533 | __entry->rcuname, __entry->rhp, __entry->func, |
534 | __entry->qlen) |
535 | ); |
536 | |
537 | TRACE_EVENT_RCU(rcu_segcb_stats, |
538 | |
539 | TP_PROTO(struct rcu_segcblist *rs, const char *ctx), |
540 | |
541 | TP_ARGS(rs, ctx), |
542 | |
543 | TP_STRUCT__entry( |
544 | __field(const char *, ctx) |
545 | __array(unsigned long, gp_seq, RCU_CBLIST_NSEGS) |
546 | __array(long, seglen, RCU_CBLIST_NSEGS) |
547 | ), |
548 | |
549 | TP_fast_assign( |
550 | __entry->ctx = ctx; |
551 | memcpy(__entry->seglen, rs->seglen, RCU_CBLIST_NSEGS * sizeof(long)); |
552 | memcpy(__entry->gp_seq, rs->gp_seq, RCU_CBLIST_NSEGS * sizeof(unsigned long)); |
553 | |
554 | ), |
555 | |
556 | TP_printk("%s seglen: (DONE=%ld, WAIT=%ld, NEXT_READY=%ld, NEXT=%ld) " |
557 | "gp_seq: (DONE=%lu, WAIT=%lu, NEXT_READY=%lu, NEXT=%lu)" , __entry->ctx, |
558 | __entry->seglen[0], __entry->seglen[1], __entry->seglen[2], __entry->seglen[3], |
559 | __entry->gp_seq[0], __entry->gp_seq[1], __entry->gp_seq[2], __entry->gp_seq[3]) |
560 | |
561 | ); |
562 | |
563 | /* |
564 | * Tracepoint for the registration of a single RCU callback of the special |
565 | * kvfree() form. The first argument is the RCU type, the second argument |
566 | * is a pointer to the RCU callback, the third argument is the offset |
567 | * of the callback within the enclosing RCU-protected data structure, |
568 | * the fourth argument is the number of lazy callbacks queued, and the |
569 | * fifth argument is the total number of callbacks queued. |
570 | */ |
571 | TRACE_EVENT_RCU(rcu_kvfree_callback, |
572 | |
573 | TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset, |
574 | long qlen), |
575 | |
576 | TP_ARGS(rcuname, rhp, offset, qlen), |
577 | |
578 | TP_STRUCT__entry( |
579 | __field(const char *, rcuname) |
580 | __field(void *, rhp) |
581 | __field(unsigned long, offset) |
582 | __field(long, qlen) |
583 | ), |
584 | |
585 | TP_fast_assign( |
586 | __entry->rcuname = rcuname; |
587 | __entry->rhp = rhp; |
588 | __entry->offset = offset; |
589 | __entry->qlen = qlen; |
590 | ), |
591 | |
592 | TP_printk("%s rhp=%p func=%ld %ld" , |
593 | __entry->rcuname, __entry->rhp, __entry->offset, |
594 | __entry->qlen) |
595 | ); |
596 | |
597 | /* |
598 | * Tracepoint for marking the beginning rcu_do_batch, performed to start |
599 | * RCU callback invocation. The first argument is the RCU flavor, |
600 | * the second is the number of lazy callbacks queued, the third is |
601 | * the total number of callbacks queued, and the fourth argument is |
602 | * the current RCU-callback batch limit. |
603 | */ |
604 | TRACE_EVENT_RCU(rcu_batch_start, |
605 | |
606 | TP_PROTO(const char *rcuname, long qlen, long blimit), |
607 | |
608 | TP_ARGS(rcuname, qlen, blimit), |
609 | |
610 | TP_STRUCT__entry( |
611 | __field(const char *, rcuname) |
612 | __field(long, qlen) |
613 | __field(long, blimit) |
614 | ), |
615 | |
616 | TP_fast_assign( |
617 | __entry->rcuname = rcuname; |
618 | __entry->qlen = qlen; |
619 | __entry->blimit = blimit; |
620 | ), |
621 | |
622 | TP_printk("%s CBs=%ld bl=%ld" , |
623 | __entry->rcuname, __entry->qlen, __entry->blimit) |
624 | ); |
625 | |
626 | /* |
627 | * Tracepoint for the invocation of a single RCU callback function. |
628 | * The first argument is the type of RCU, and the second argument is |
629 | * a pointer to the RCU callback itself. |
630 | */ |
631 | TRACE_EVENT_RCU(rcu_invoke_callback, |
632 | |
633 | TP_PROTO(const char *rcuname, struct rcu_head *rhp), |
634 | |
635 | TP_ARGS(rcuname, rhp), |
636 | |
637 | TP_STRUCT__entry( |
638 | __field(const char *, rcuname) |
639 | __field(void *, rhp) |
640 | __field(void *, func) |
641 | ), |
642 | |
643 | TP_fast_assign( |
644 | __entry->rcuname = rcuname; |
645 | __entry->rhp = rhp; |
646 | __entry->func = rhp->func; |
647 | ), |
648 | |
649 | TP_printk("%s rhp=%p func=%ps" , |
650 | __entry->rcuname, __entry->rhp, __entry->func) |
651 | ); |
652 | |
653 | /* |
654 | * Tracepoint for the invocation of a single RCU callback of the special |
655 | * kvfree() form. The first argument is the RCU flavor, the second |
656 | * argument is a pointer to the RCU callback, and the third argument |
657 | * is the offset of the callback within the enclosing RCU-protected |
658 | * data structure. |
659 | */ |
660 | TRACE_EVENT_RCU(rcu_invoke_kvfree_callback, |
661 | |
662 | TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset), |
663 | |
664 | TP_ARGS(rcuname, rhp, offset), |
665 | |
666 | TP_STRUCT__entry( |
667 | __field(const char *, rcuname) |
668 | __field(void *, rhp) |
669 | __field(unsigned long, offset) |
670 | ), |
671 | |
672 | TP_fast_assign( |
673 | __entry->rcuname = rcuname; |
674 | __entry->rhp = rhp; |
675 | __entry->offset = offset; |
676 | ), |
677 | |
678 | TP_printk("%s rhp=%p func=%ld" , |
679 | __entry->rcuname, __entry->rhp, __entry->offset) |
680 | ); |
681 | |
682 | /* |
683 | * Tracepoint for the invocation of a single RCU callback of the special |
684 | * kfree_bulk() form. The first argument is the RCU flavor, the second |
685 | * argument is a number of elements in array to free, the third is an |
686 | * address of the array holding nr_records entries. |
687 | */ |
688 | TRACE_EVENT_RCU(rcu_invoke_kfree_bulk_callback, |
689 | |
690 | TP_PROTO(const char *rcuname, unsigned long nr_records, void **p), |
691 | |
692 | TP_ARGS(rcuname, nr_records, p), |
693 | |
694 | TP_STRUCT__entry( |
695 | __field(const char *, rcuname) |
696 | __field(unsigned long, nr_records) |
697 | __field(void **, p) |
698 | ), |
699 | |
700 | TP_fast_assign( |
701 | __entry->rcuname = rcuname; |
702 | __entry->nr_records = nr_records; |
703 | __entry->p = p; |
704 | ), |
705 | |
706 | TP_printk("%s bulk=0x%p nr_records=%lu" , |
707 | __entry->rcuname, __entry->p, __entry->nr_records) |
708 | ); |
709 | |
710 | /* |
711 | * Tracepoint for exiting rcu_do_batch after RCU callbacks have been |
712 | * invoked. The first argument is the name of the RCU flavor, |
713 | * the second argument is number of callbacks actually invoked, |
714 | * the third argument (cb) is whether or not any of the callbacks that |
715 | * were ready to invoke at the beginning of this batch are still |
716 | * queued, the fourth argument (nr) is the return value of need_resched(), |
717 | * the fifth argument (iit) is 1 if the current task is the idle task, |
718 | * and the sixth argument (risk) is the return value from |
719 | * rcu_is_callbacks_kthread(). |
720 | */ |
721 | TRACE_EVENT_RCU(rcu_batch_end, |
722 | |
723 | TP_PROTO(const char *rcuname, int callbacks_invoked, |
724 | char cb, char nr, char iit, char risk), |
725 | |
726 | TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk), |
727 | |
728 | TP_STRUCT__entry( |
729 | __field(const char *, rcuname) |
730 | __field(int, callbacks_invoked) |
731 | __field(char, cb) |
732 | __field(char, nr) |
733 | __field(char, iit) |
734 | __field(char, risk) |
735 | ), |
736 | |
737 | TP_fast_assign( |
738 | __entry->rcuname = rcuname; |
739 | __entry->callbacks_invoked = callbacks_invoked; |
740 | __entry->cb = cb; |
741 | __entry->nr = nr; |
742 | __entry->iit = iit; |
743 | __entry->risk = risk; |
744 | ), |
745 | |
746 | TP_printk("%s CBs-invoked=%d idle=%c%c%c%c" , |
747 | __entry->rcuname, __entry->callbacks_invoked, |
748 | __entry->cb ? 'C' : '.', |
749 | __entry->nr ? 'S' : '.', |
750 | __entry->iit ? 'I' : '.', |
751 | __entry->risk ? 'R' : '.') |
752 | ); |
753 | |
754 | /* |
755 | * Tracepoint for rcutorture readers. The first argument is the name |
756 | * of the RCU flavor from rcutorture's viewpoint and the second argument |
757 | * is the callback address. The third argument is the start time in |
758 | * seconds, and the last two arguments are the grace period numbers |
759 | * at the beginning and end of the read, respectively. Note that the |
760 | * callback address can be NULL. |
761 | */ |
762 | #define RCUTORTURENAME_LEN 8 |
763 | TRACE_EVENT_RCU(rcu_torture_read, |
764 | |
765 | TP_PROTO(const char *rcutorturename, struct rcu_head *rhp, |
766 | unsigned long secs, unsigned long c_old, unsigned long c), |
767 | |
768 | TP_ARGS(rcutorturename, rhp, secs, c_old, c), |
769 | |
770 | TP_STRUCT__entry( |
771 | __array(char, rcutorturename, RCUTORTURENAME_LEN) |
772 | __field(struct rcu_head *, rhp) |
773 | __field(unsigned long, secs) |
774 | __field(unsigned long, c_old) |
775 | __field(unsigned long, c) |
776 | ), |
777 | |
778 | TP_fast_assign( |
779 | strscpy(__entry->rcutorturename, rcutorturename, RCUTORTURENAME_LEN); |
780 | __entry->rhp = rhp; |
781 | __entry->secs = secs; |
782 | __entry->c_old = c_old; |
783 | __entry->c = c; |
784 | ), |
785 | |
786 | TP_printk("%s torture read %p %luus c: %lu %lu" , |
787 | __entry->rcutorturename, __entry->rhp, |
788 | __entry->secs, __entry->c_old, __entry->c) |
789 | ); |
790 | |
791 | /* |
792 | * Tracepoint for rcu_barrier() execution. The string "s" describes |
793 | * the rcu_barrier phase: |
794 | * "Begin": rcu_barrier() started. |
795 | * "CB": An rcu_barrier_callback() invoked a callback, not the last. |
796 | * "EarlyExit": rcu_barrier() piggybacked, thus early exit. |
797 | * "Inc1": rcu_barrier() piggyback check counter incremented. |
798 | * "Inc2": rcu_barrier() piggyback check counter incremented. |
799 | * "IRQ": An rcu_barrier_callback() callback posted on remote CPU. |
800 | * "IRQNQ": An rcu_barrier_callback() callback found no callbacks. |
801 | * "LastCB": An rcu_barrier_callback() invoked the last callback. |
802 | * "NQ": rcu_barrier() found a CPU with no callbacks. |
803 | * "OnlineQ": rcu_barrier() found online CPU with callbacks. |
804 | * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument |
805 | * is the count of remaining callbacks, and "done" is the piggybacking count. |
806 | */ |
807 | TRACE_EVENT_RCU(rcu_barrier, |
808 | |
809 | TP_PROTO(const char *rcuname, const char *s, int cpu, int cnt, unsigned long done), |
810 | |
811 | TP_ARGS(rcuname, s, cpu, cnt, done), |
812 | |
813 | TP_STRUCT__entry( |
814 | __field(const char *, rcuname) |
815 | __field(const char *, s) |
816 | __field(int, cpu) |
817 | __field(int, cnt) |
818 | __field(unsigned long, done) |
819 | ), |
820 | |
821 | TP_fast_assign( |
822 | __entry->rcuname = rcuname; |
823 | __entry->s = s; |
824 | __entry->cpu = cpu; |
825 | __entry->cnt = cnt; |
826 | __entry->done = done; |
827 | ), |
828 | |
829 | TP_printk("%s %s cpu %d remaining %d # %lu" , |
830 | __entry->rcuname, __entry->s, __entry->cpu, __entry->cnt, |
831 | __entry->done) |
832 | ); |
833 | |
834 | #endif /* _TRACE_RCU_H */ |
835 | |
836 | /* This part must be outside protection */ |
837 | #include <trace/define_trace.h> |
838 | |