1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Common code for probe-based Dynamic events.
4	*
5	* This code was copied from kernel/trace/trace_kprobe.c written by
6	* Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
7	*
8	* Updates to make this generic:
9	* Copyright (C) IBM Corporation, 2010-2011
10	* Author: Srikar Dronamraju
11	*/
12	#define pr_fmt(fmt) "trace_probe: " fmt
13
14	#include <linux/bpf.h>
15	#include "trace_btf.h"
16
17	#include "trace_probe.h"
18
19	#undef C
20	#define C(a, b) b
21
22	static const char *trace_probe_err_text[] = { ERRORS };
23
24	static const char *reserved_field_names[] = {
25	"common_type",
26	"common_flags",
27	"common_preempt_count",
28	"common_pid",
29	"common_tgid",
30	FIELD_STRING_IP,
31	FIELD_STRING_RETIP,
32	FIELD_STRING_FUNC,
33	};
34
35	/* Printing in basic type function template */
36	#define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt) \
37	int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
38	{ \
39	trace_seq_printf(s, fmt, *(type *)data); \
40	return !trace_seq_has_overflowed(s); \
41	} \
42	const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
43
44	DEFINE_BASIC_PRINT_TYPE_FUNC(u8, u8, "%u")
45	DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
46	DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
47	DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
48	DEFINE_BASIC_PRINT_TYPE_FUNC(s8, s8, "%d")
49	DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
50	DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
51	DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
52	DEFINE_BASIC_PRINT_TYPE_FUNC(x8, u8, "0x%x")
53	DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
54	DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
55	DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
56	DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
57
58	int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
59	{
60	trace_seq_printf(s, fmt: "%pS", (void *)*(unsigned long *)data);
61	return !trace_seq_has_overflowed(s);
62	}
63	const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
64
65	/* Print type function for string type */
66	int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
67	{
68	int len = *(u32 *)data >> 16;
69
70	if (!len)
71	trace_seq_puts(s, FAULT_STRING);
72	else
73	trace_seq_printf(s, fmt: "\"%s\"",
74	(const char *)get_loc_data(dl: data, ent));
75	return !trace_seq_has_overflowed(s);
76	}
77
78	const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
79
80	/* Fetch type information table */
81	static const struct fetch_type probe_fetch_types[] = {
82	/* Special types */
83	__ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
84	"__data_loc char[]"),
85	__ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
86	"__data_loc char[]"),
87	__ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
88	"__data_loc char[]"),
89	/* Basic types */
90	ASSIGN_FETCH_TYPE(u8, u8, 0),
91	ASSIGN_FETCH_TYPE(u16, u16, 0),
92	ASSIGN_FETCH_TYPE(u32, u32, 0),
93	ASSIGN_FETCH_TYPE(u64, u64, 0),
94	ASSIGN_FETCH_TYPE(s8, u8, 1),
95	ASSIGN_FETCH_TYPE(s16, u16, 1),
96	ASSIGN_FETCH_TYPE(s32, u32, 1),
97	ASSIGN_FETCH_TYPE(s64, u64, 1),
98	ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
99	ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
100	ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
101	ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
102	ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8, 0),
103	ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
104
105	ASSIGN_FETCH_TYPE_END
106	};
107
108	static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
109	{
110	int i;
111
112	/* Reject the symbol/symstr for uprobes */
113	if (type && (flags & TPARG_FL_USER) &&
114	(!strcmp(type, "symbol") || !strcmp(type, "symstr")))
115	return NULL;
116
117	if (!type)
118	type = DEFAULT_FETCH_TYPE_STR;
119
120	/* Special case: bitfield */
121	if (*type == 'b') {
122	unsigned long bs;
123
124	type = strchr(type, '/');
125	if (!type)
126	goto fail;
127
128	type++;
129	if (kstrtoul(s: type, base: 0, res: &bs))
130	goto fail;
131
132	switch (bs) {
133	case 8:
134	return find_fetch_type(type: "u8", flags);
135	case 16:
136	return find_fetch_type(type: "u16", flags);
137	case 32:
138	return find_fetch_type(type: "u32", flags);
139	case 64:
140	return find_fetch_type(type: "u64", flags);
141	default:
142	goto fail;
143	}
144	}
145
146	for (i = 0; probe_fetch_types[i].name; i++) {
147	if (strcmp(type, probe_fetch_types[i].name) == 0)
148	return &probe_fetch_types[i];
149	}
150
151	fail:
152	return NULL;
153	}
154
155	static struct trace_probe_log trace_probe_log;
156
157	void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
158	{
159	trace_probe_log.subsystem = subsystem;
160	trace_probe_log.argc = argc;
161	trace_probe_log.argv = argv;
162	trace_probe_log.index = 0;
163	}
164
165	void trace_probe_log_clear(void)
166	{
167	memset(&trace_probe_log, 0, sizeof(trace_probe_log));
168	}
169
170	void trace_probe_log_set_index(int index)
171	{
172	trace_probe_log.index = index;
173	}
174
175	void __trace_probe_log_err(int offset, int err_type)
176	{
177	char *command, *p;
178	int i, len = 0, pos = 0;
179
180	if (!trace_probe_log.argv)
181	return;
182
183	/* Recalculate the length and allocate buffer */
184	for (i = 0; i < trace_probe_log.argc; i++) {
185	if (i == trace_probe_log.index)
186	pos = len;
187	len += strlen(trace_probe_log.argv[i]) + 1;
188	}
189	command = kzalloc(size: len, GFP_KERNEL);
190	if (!command)
191	return;
192
193	if (trace_probe_log.index >= trace_probe_log.argc) {
194	/**
195	* Set the error position is next to the last arg + space.
196	* Note that len includes the terminal null and the cursor
197	* appears at pos + 1.
198	*/
199	pos = len;
200	offset = 0;
201	}
202
203	/* And make a command string from argv array */
204	p = command;
205	for (i = 0; i < trace_probe_log.argc; i++) {
206	len = strlen(trace_probe_log.argv[i]);
207	strcpy(p, q: trace_probe_log.argv[i]);
208	p[len] = ' ';
209	p += len + 1;
210	}
211	*(p - 1) = '\0';
212
213	tracing_log_err(NULL, loc: trace_probe_log.subsystem, cmd: command,
214	errs: trace_probe_err_text, type: err_type, pos: pos + offset);
215
216	kfree(objp: command);
217	}
218
219	/* Split symbol and offset. */
220	int traceprobe_split_symbol_offset(char *symbol, long *offset)
221	{
222	char *tmp;
223	int ret;
224
225	if (!offset)
226	return -EINVAL;
227
228	tmp = strpbrk(symbol, "+-");
229	if (tmp) {
230	ret = kstrtol(s: tmp, base: 0, res: offset);
231	if (ret)
232	return ret;
233	*tmp = '\0';
234	} else
235	*offset = 0;
236
237	return 0;
238	}
239
240	/* @buf must has MAX_EVENT_NAME_LEN size */
241	int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
242	char *buf, int offset)
243	{
244	const char *slash, *event = *pevent;
245	int len;
246
247	slash = strchr(event, '/');
248	if (!slash)
249	slash = strchr(event, '.');
250
251	if (slash) {
252	if (slash == event) {
253	trace_probe_log_err(offset, NO_GROUP_NAME);
254	return -EINVAL;
255	}
256	if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
257	trace_probe_log_err(offset, GROUP_TOO_LONG);
258	return -EINVAL;
259	}
260	strscpy(p: buf, q: event, size: slash - event + 1);
261	if (!is_good_system_name(name: buf)) {
262	trace_probe_log_err(offset, BAD_GROUP_NAME);
263	return -EINVAL;
264	}
265	*pgroup = buf;
266	*pevent = slash + 1;
267	offset += slash - event + 1;
268	event = *pevent;
269	}
270	len = strlen(event);
271	if (len == 0) {
272	if (slash) {
273	*pevent = NULL;
274	return 0;
275	}
276	trace_probe_log_err(offset, NO_EVENT_NAME);
277	return -EINVAL;
278	} else if (len > MAX_EVENT_NAME_LEN) {
279	trace_probe_log_err(offset, EVENT_TOO_LONG);
280	return -EINVAL;
281	}
282	if (!is_good_name(name: event)) {
283	trace_probe_log_err(offset, BAD_EVENT_NAME);
284	return -EINVAL;
285	}
286	return 0;
287	}
288
289	static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
290	struct traceprobe_parse_context *ctx)
291	{
292	struct ftrace_event_field *field;
293	struct list_head *head;
294
295	head = trace_get_fields(event_call: ctx->event);
296	list_for_each_entry(field, head, link) {
297	if (!strcmp(arg, field->name)) {
298	code->op = FETCH_OP_TP_ARG;
299	code->data = field;
300	return 0;
301	}
302	}
303	return -ENOENT;
304	}
305
306	#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
307
308	static u32 btf_type_int(const struct btf_type *t)
309	{
310	return *(u32 *)(t + 1);
311	}
312
313	static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
314	{
315	const struct btf_type *real_type;
316	u32 intdata;
317	s32 tid;
318
319	real_type = btf_type_skip_modifiers(btf, type->type, &tid);
320	if (!real_type)
321	return false;
322
323	if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
324	return false;
325
326	intdata = btf_type_int(real_type);
327	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
328	&& BTF_INT_BITS(intdata) == 8;
329	}
330
331	static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
332	{
333	const struct btf_type *real_type;
334	const struct btf_array *array;
335	u32 intdata;
336	s32 tid;
337
338	if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
339	return false;
340
341	array = (const struct btf_array *)(type + 1);
342
343	real_type = btf_type_skip_modifiers(btf, array->type, &tid);
344
345	intdata = btf_type_int(real_type);
346	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
347	&& BTF_INT_BITS(intdata) == 8;
348	}
349
350	static int check_prepare_btf_string_fetch(char *typename,
351	struct fetch_insn **pcode,
352	struct traceprobe_parse_context *ctx)
353	{
354	struct btf *btf = ctx->btf;
355
356	if (!btf || !ctx->last_type)
357	return 0;
358
359	/* char [] does not need any change. */
360	if (btf_type_is_char_array(btf, ctx->last_type))
361	return 0;
362
363	/* char * requires dereference the pointer. */
364	if (btf_type_is_char_ptr(btf, ctx->last_type)) {
365	struct fetch_insn *code = *pcode + 1;
366
367	if (code->op == FETCH_OP_END) {
368	trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
369	return -E2BIG;
370	}
371	if (typename[0] == 'u')
372	code->op = FETCH_OP_UDEREF;
373	else
374	code->op = FETCH_OP_DEREF;
375	code->offset = 0;
376	*pcode = code;
377	return 0;
378	}
379	/* Other types are not available for string */
380	trace_probe_log_err(ctx->offset, BAD_TYPE4STR);
381	return -EINVAL;
382	}
383
384	static const char *fetch_type_from_btf_type(struct btf *btf,
385	const struct btf_type *type,
386	struct traceprobe_parse_context *ctx)
387	{
388	u32 intdata;
389
390	/* TODO: const char * could be converted as a string */
391	switch (BTF_INFO_KIND(type->info)) {
392	case BTF_KIND_ENUM:
393	/* enum is "int", so convert to "s32" */
394	return "s32";
395	case BTF_KIND_ENUM64:
396	return "s64";
397	case BTF_KIND_PTR:
398	/* pointer will be converted to "x??" */
399	if (IS_ENABLED(CONFIG_64BIT))
400	return "x64";
401	else
402	return "x32";
403	case BTF_KIND_INT:
404	intdata = btf_type_int(type);
405	if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
406	switch (BTF_INT_BITS(intdata)) {
407	case 8:
408	return "s8";
409	case 16:
410	return "s16";
411	case 32:
412	return "s32";
413	case 64:
414	return "s64";
415	}
416	} else { /* unsigned */
417	switch (BTF_INT_BITS(intdata)) {
418	case 8:
419	return "u8";
420	case 16:
421	return "u16";
422	case 32:
423	return "u32";
424	case 64:
425	return "u64";
426	}
427	/* bitfield, size is encoded in the type */
428	ctx->last_bitsize = BTF_INT_BITS(intdata);
429	ctx->last_bitoffs += BTF_INT_OFFSET(intdata);
430	return "u64";
431	}
432	}
433	/* TODO: support other types */
434
435	return NULL;
436	}
437
438	static int query_btf_context(struct traceprobe_parse_context *ctx)
439	{
440	const struct btf_param *param;
441	const struct btf_type *type;
442	struct btf *btf;
443	s32 nr;
444
445	if (ctx->btf)
446	return 0;
447
448	if (!ctx->funcname)
449	return -EINVAL;
450
451	type = btf_find_func_proto(ctx->funcname, &btf);
452	if (!type)
453	return -ENOENT;
454
455	ctx->btf = btf;
456	ctx->proto = type;
457
458	/* ctx->params is optional, since func(void) will not have params. */
459	nr = 0;
460	param = btf_get_func_param(type, &nr);
461	if (!IS_ERR_OR_NULL(param)) {
462	/* Hide the first 'data' argument of tracepoint */
463	if (ctx->flags & TPARG_FL_TPOINT) {
464	nr--;
465	param++;
466	}
467	}
468
469	if (nr > 0) {
470	ctx->nr_params = nr;
471	ctx->params = param;
472	} else {
473	ctx->nr_params = 0;
474	ctx->params = NULL;
475	}
476
477	return 0;
478	}
479
480	static void clear_btf_context(struct traceprobe_parse_context *ctx)
481	{
482	if (ctx->btf) {
483	btf_put(ctx->btf);
484	ctx->btf = NULL;
485	ctx->proto = NULL;
486	ctx->params = NULL;
487	ctx->nr_params = 0;
488	}
489	}
490
491	/* Return 1 if the field separater is arrow operator ('->') */
492	static int split_next_field(char *varname, char **next_field,
493	struct traceprobe_parse_context *ctx)
494	{
495	char *field;
496	int ret = 0;
497
498	field = strpbrk(varname, ".-");
499	if (field) {
500	if (field[0] == '-' && field[1] == '>') {
501	field[0] = '\0';
502	field += 2;
503	ret = 1;
504	} else if (field[0] == '.') {
505	field[0] = '\0';
506	field += 1;
507	} else {
508	trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN);
509	return -EINVAL;
510	}
511	*next_field = field;
512	}
513
514	return ret;
515	}
516
517	/*
518	* Parse the field of data structure. The @type must be a pointer type
519	* pointing the target data structure type.
520	*/
521	static int parse_btf_field(char *fieldname, const struct btf_type *type,
522	struct fetch_insn **pcode, struct fetch_insn *end,
523	struct traceprobe_parse_context *ctx)
524	{
525	struct fetch_insn *code = *pcode;
526	const struct btf_member *field;
527	u32 bitoffs, anon_offs;
528	char *next;
529	int is_ptr;
530	s32 tid;
531
532	do {
533	/* Outer loop for solving arrow operator ('->') */
534	if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
535	trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
536	return -EINVAL;
537	}
538	/* Convert a struct pointer type to a struct type */
539	type = btf_type_skip_modifiers(ctx->btf, type->type, &tid);
540	if (!type) {
541	trace_probe_log_err(ctx->offset, BAD_BTF_TID);
542	return -EINVAL;
543	}
544
545	bitoffs = 0;
546	do {
547	/* Inner loop for solving dot operator ('.') */
548	next = NULL;
549	is_ptr = split_next_field(fieldname, &next, ctx);
550	if (is_ptr < 0)
551	return is_ptr;
552
553	anon_offs = 0;
554	field = btf_find_struct_member(ctx->btf, type, fieldname,
555	&anon_offs);
556	if (!field) {
557	trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
558	return -ENOENT;
559	}
560	/* Add anonymous structure/union offset */
561	bitoffs += anon_offs;
562
563	/* Accumulate the bit-offsets of the dot-connected fields */
564	if (btf_type_kflag(type)) {
565	bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
566	ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
567	} else {
568	bitoffs += field->offset;
569	ctx->last_bitsize = 0;
570	}
571
572	type = btf_type_skip_modifiers(ctx->btf, field->type, &tid);
573	if (!type) {
574	trace_probe_log_err(ctx->offset, BAD_BTF_TID);
575	return -EINVAL;
576	}
577
578	ctx->offset += next - fieldname;
579	fieldname = next;
580	} while (!is_ptr && fieldname);
581
582	if (++code == end) {
583	trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
584	return -EINVAL;
585	}
586	code->op = FETCH_OP_DEREF; /* TODO: user deref support */
587	code->offset = bitoffs / 8;
588	*pcode = code;
589
590	ctx->last_bitoffs = bitoffs % 8;
591	ctx->last_type = type;
592	} while (fieldname);
593
594	return 0;
595	}
596
597	static int parse_btf_arg(char *varname,
598	struct fetch_insn **pcode, struct fetch_insn *end,
599	struct traceprobe_parse_context *ctx)
600	{
601	struct fetch_insn *code = *pcode;
602	const struct btf_param *params;
603	const struct btf_type *type;
604	char *field = NULL;
605	int i, is_ptr, ret;
606	u32 tid;
607
608	if (WARN_ON_ONCE(!ctx->funcname))
609	return -EINVAL;
610
611	is_ptr = split_next_field(varname, &field, ctx);
612	if (is_ptr < 0)
613	return is_ptr;
614	if (!is_ptr && field) {
615	/* dot-connected field on an argument is not supported. */
616	trace_probe_log_err(ctx->offset + field - varname,
617	NOSUP_DAT_ARG);
618	return -EOPNOTSUPP;
619	}
620
621	if (ctx->flags & TPARG_FL_RETURN) {
622	if (strcmp(varname, "$retval") != 0) {
623	trace_probe_log_err(ctx->offset, NO_BTFARG);
624	return -ENOENT;
625	}
626	code->op = FETCH_OP_RETVAL;
627	/* Check whether the function return type is not void */
628	if (query_btf_context(ctx) == 0) {
629	if (ctx->proto->type == 0) {
630	trace_probe_log_err(ctx->offset, NO_RETVAL);
631	return -ENOENT;
632	}
633	tid = ctx->proto->type;
634	goto found;
635	}
636	if (field) {
637	trace_probe_log_err(ctx->offset + field - varname,
638	NO_BTF_ENTRY);
639	return -ENOENT;
640	}
641	return 0;
642	}
643
644	if (!ctx->btf) {
645	ret = query_btf_context(ctx);
646	if (ret < 0 || ctx->nr_params == 0) {
647	trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
648	return PTR_ERR(params);
649	}
650	}
651	params = ctx->params;
652
653	for (i = 0; i < ctx->nr_params; i++) {
654	const char *name = btf_name_by_offset(ctx->btf, params[i].name_off);
655
656	if (name && !strcmp(name, varname)) {
657	code->op = FETCH_OP_ARG;
658	if (ctx->flags & TPARG_FL_TPOINT)
659	code->param = i + 1;
660	else
661	code->param = i;
662	tid = params[i].type;
663	goto found;
664	}
665	}
666	trace_probe_log_err(ctx->offset, NO_BTFARG);
667	return -ENOENT;
668
669	found:
670	type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
671	if (!type) {
672	trace_probe_log_err(ctx->offset, BAD_BTF_TID);
673	return -EINVAL;
674	}
675	/* Initialize the last type information */
676	ctx->last_type = type;
677	ctx->last_bitoffs = 0;
678	ctx->last_bitsize = 0;
679	if (field) {
680	ctx->offset += field - varname;
681	return parse_btf_field(field, type, pcode, end, ctx);
682	}
683	return 0;
684	}
685
686	static const struct fetch_type *find_fetch_type_from_btf_type(
687	struct traceprobe_parse_context *ctx)
688	{
689	struct btf *btf = ctx->btf;
690	const char *typestr = NULL;
691
692	if (btf && ctx->last_type)
693	typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx);
694
695	return find_fetch_type(typestr, ctx->flags);
696	}
697
698	static int parse_btf_bitfield(struct fetch_insn **pcode,
699	struct traceprobe_parse_context *ctx)
700	{
701	struct fetch_insn *code = *pcode;
702
703	if ((ctx->last_bitsize % 8 == 0) && ctx->last_bitoffs == 0)
704	return 0;
705
706	code++;
707	if (code->op != FETCH_OP_NOP) {
708	trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
709	return -EINVAL;
710	}
711	*pcode = code;
712
713	code->op = FETCH_OP_MOD_BF;
714	code->lshift = 64 - (ctx->last_bitsize + ctx->last_bitoffs);
715	code->rshift = 64 - ctx->last_bitsize;
716	code->basesize = 64 / 8;
717	return 0;
718	}
719
720	#else
721	static void clear_btf_context(struct traceprobe_parse_context *ctx)
722	{
723	ctx->btf = NULL;
724	}
725
726	static int query_btf_context(struct traceprobe_parse_context *ctx)
727	{
728	return -EOPNOTSUPP;
729	}
730
731	static int parse_btf_arg(char *varname,
732	struct fetch_insn **pcode, struct fetch_insn *end,
733	struct traceprobe_parse_context *ctx)
734	{
735	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
736	return -EOPNOTSUPP;
737	}
738
739	static int parse_btf_bitfield(struct fetch_insn **pcode,
740	struct traceprobe_parse_context *ctx)
741	{
742	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
743	return -EOPNOTSUPP;
744	}
745
746	#define find_fetch_type_from_btf_type(ctx) \
747	find_fetch_type(NULL, ctx->flags)
748
749	static int check_prepare_btf_string_fetch(char *typename,
750	struct fetch_insn **pcode,
751	struct traceprobe_parse_context *ctx)
752	{
753	return 0;
754	}
755
756	#endif
757
758	#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
759
760	/* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
761	static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
762	struct fetch_insn **pcode,
763	struct fetch_insn *end,
764	struct traceprobe_parse_context *ctx)
765	{
766	struct fetch_insn *code = *pcode;
767	int err = TP_ERR_BAD_VAR;
768	char *arg = orig_arg + 1;
769	unsigned long param;
770	int ret = 0;
771	int len;
772
773	if (ctx->flags & TPARG_FL_TEVENT) {
774	if (code->data)
775	return -EFAULT;
776	ret = parse_trace_event_arg(arg, code, ctx);
777	if (!ret)
778	return 0;
779	if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
780	code->op = FETCH_OP_COMM;
781	return 0;
782	}
783	/* backward compatibility */
784	ctx->offset = 0;
785	goto inval;
786	}
787
788	if (str_has_prefix(str: arg, prefix: "retval")) {
789	if (!(ctx->flags & TPARG_FL_RETURN)) {
790	err = TP_ERR_RETVAL_ON_PROBE;
791	goto inval;
792	}
793	if (!(ctx->flags & TPARG_FL_KERNEL) ||
794	!IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
795	code->op = FETCH_OP_RETVAL;
796	return 0;
797	}
798	return parse_btf_arg(varname: orig_arg, pcode, end, ctx);
799	}
800
801	len = str_has_prefix(str: arg, prefix: "stack");
802	if (len) {
803
804	if (arg[len] == '\0') {
805	code->op = FETCH_OP_STACKP;
806	return 0;
807	}
808
809	if (isdigit(c: arg[len])) {
810	ret = kstrtoul(s: arg + len, base: 10, res: &param);
811	if (ret)
812	goto inval;
813
814	if ((ctx->flags & TPARG_FL_KERNEL) &&
815	param > PARAM_MAX_STACK) {
816	err = TP_ERR_BAD_STACK_NUM;
817	goto inval;
818	}
819	code->op = FETCH_OP_STACK;
820	code->param = (unsigned int)param;
821	return 0;
822	}
823	goto inval;
824	}
825
826	if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
827	code->op = FETCH_OP_COMM;
828	return 0;
829	}
830
831	#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
832	len = str_has_prefix(str: arg, prefix: "arg");
833	if (len && tparg_is_function_entry(flags: ctx->flags)) {
834	ret = kstrtoul(s: arg + len, base: 10, res: &param);
835	if (ret)
836	goto inval;
837
838	if (!param || param > PARAM_MAX_STACK) {
839	err = TP_ERR_BAD_ARG_NUM;
840	goto inval;
841	}
842
843	code->op = FETCH_OP_ARG;
844	code->param = (unsigned int)param - 1;
845	/*
846	* The tracepoint probe will probe a stub function, and the
847	* first parameter of the stub is a dummy and should be ignored.
848	*/
849	if (ctx->flags & TPARG_FL_TPOINT)
850	code->param++;
851	return 0;
852	}
853	#endif
854
855	inval:
856	__trace_probe_log_err(offset: ctx->offset, err_type: err);
857	return -EINVAL;
858	}
859
860	static int str_to_immediate(char *str, unsigned long *imm)
861	{
862	if (isdigit(c: str[0]))
863	return kstrtoul(s: str, base: 0, res: imm);
864	else if (str[0] == '-')
865	return kstrtol(s: str, base: 0, res: (long *)imm);
866	else if (str[0] == '+')
867	return kstrtol(s: str + 1, base: 0, res: (long *)imm);
868	return -EINVAL;
869	}
870
871	static int __parse_imm_string(char *str, char **pbuf, int offs)
872	{
873	size_t len = strlen(str);
874
875	if (str[len - 1] != '"') {
876	trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
877	return -EINVAL;
878	}
879	*pbuf = kstrndup(s: str, len: len - 1, GFP_KERNEL);
880	if (!*pbuf)
881	return -ENOMEM;
882	return 0;
883	}
884
885	/* Recursive argument parser */
886	static int
887	parse_probe_arg(char *arg, const struct fetch_type *type,
888	struct fetch_insn **pcode, struct fetch_insn *end,
889	struct traceprobe_parse_context *ctx)
890	{
891	struct fetch_insn *code = *pcode;
892	unsigned long param;
893	int deref = FETCH_OP_DEREF;
894	long offset = 0;
895	char *tmp;
896	int ret = 0;
897
898	switch (arg[0]) {
899	case '$':
900	ret = parse_probe_vars(orig_arg: arg, t: type, pcode, end, ctx);
901	break;
902
903	case '%': /* named register */
904	if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
905	/* eprobe and fprobe do not handle registers */
906	trace_probe_log_err(ctx->offset, BAD_VAR);
907	break;
908	}
909	ret = regs_query_register_offset(name: arg + 1);
910	if (ret >= 0) {
911	code->op = FETCH_OP_REG;
912	code->param = (unsigned int)ret;
913	ret = 0;
914	} else
915	trace_probe_log_err(ctx->offset, BAD_REG_NAME);
916	break;
917
918	case '@': /* memory, file-offset or symbol */
919	if (isdigit(c: arg[1])) {
920	ret = kstrtoul(s: arg + 1, base: 0, res: &param);
921	if (ret) {
922	trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
923	break;
924	}
925	/* load address */
926	code->op = FETCH_OP_IMM;
927	code->immediate = param;
928	} else if (arg[1] == '+') {
929	/* kprobes don't support file offsets */
930	if (ctx->flags & TPARG_FL_KERNEL) {
931	trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
932	return -EINVAL;
933	}
934	ret = kstrtol(s: arg + 2, base: 0, res: &offset);
935	if (ret) {
936	trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
937	break;
938	}
939
940	code->op = FETCH_OP_FOFFS;
941	code->immediate = (unsigned long)offset; // imm64?
942	} else {
943	/* uprobes don't support symbols */
944	if (!(ctx->flags & TPARG_FL_KERNEL)) {
945	trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
946	return -EINVAL;
947	}
948	/* Preserve symbol for updating */
949	code->op = FETCH_NOP_SYMBOL;
950	code->data = kstrdup(s: arg + 1, GFP_KERNEL);
951	if (!code->data)
952	return -ENOMEM;
953	if (++code == end) {
954	trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
955	return -EINVAL;
956	}
957	code->op = FETCH_OP_IMM;
958	code->immediate = 0;
959	}
960	/* These are fetching from memory */
961	if (++code == end) {
962	trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
963	return -EINVAL;
964	}
965	*pcode = code;
966	code->op = FETCH_OP_DEREF;
967	code->offset = offset;
968	break;
969
970	case '+': /* deref memory */
971	case '-':
972	if (arg[1] == 'u') {
973	deref = FETCH_OP_UDEREF;
974	arg[1] = arg[0];
975	arg++;
976	}
977	if (arg[0] == '+')
978	arg++; /* Skip '+', because kstrtol() rejects it. */
979	tmp = strchr(arg, '(');
980	if (!tmp) {
981	trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
982	return -EINVAL;
983	}
984	*tmp = '\0';
985	ret = kstrtol(s: arg, base: 0, res: &offset);
986	if (ret) {
987	trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
988	break;
989	}
990	ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
991	arg = tmp + 1;
992	tmp = strrchr(arg, ')');
993	if (!tmp) {
994	trace_probe_log_err(ctx->offset + strlen(arg),
995	DEREF_OPEN_BRACE);
996	return -EINVAL;
997	} else {
998	const struct fetch_type *t2 = find_fetch_type(NULL, flags: ctx->flags);
999	int cur_offs = ctx->offset;
1000
1001	*tmp = '\0';
1002	ret = parse_probe_arg(arg, type: t2, pcode: &code, end, ctx);
1003	if (ret)
1004	break;
1005	ctx->offset = cur_offs;
1006	if (code->op == FETCH_OP_COMM ||
1007	code->op == FETCH_OP_DATA) {
1008	trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
1009	return -EINVAL;
1010	}
1011	if (++code == end) {
1012	trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1013	return -EINVAL;
1014	}
1015	*pcode = code;
1016
1017	code->op = deref;
1018	code->offset = offset;
1019	/* Reset the last type if used */
1020	ctx->last_type = NULL;
1021	}
1022	break;
1023	case '\\': /* Immediate value */
1024	if (arg[1] == '"') { /* Immediate string */
1025	ret = __parse_imm_string(str: arg + 2, pbuf: &tmp, offs: ctx->offset + 2);
1026	if (ret)
1027	break;
1028	code->op = FETCH_OP_DATA;
1029	code->data = tmp;
1030	} else {
1031	ret = str_to_immediate(str: arg + 1, imm: &code->immediate);
1032	if (ret)
1033	trace_probe_log_err(ctx->offset + 1, BAD_IMM);
1034	else
1035	code->op = FETCH_OP_IMM;
1036	}
1037	break;
1038	default:
1039	if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */
1040	if (!tparg_is_function_entry(flags: ctx->flags)) {
1041	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1042	return -EINVAL;
1043	}
1044	ret = parse_btf_arg(varname: arg, pcode, end, ctx);
1045	break;
1046	}
1047	}
1048	if (!ret && code->op == FETCH_OP_NOP) {
1049	/* Parsed, but do not find fetch method */
1050	trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
1051	ret = -EINVAL;
1052	}
1053	return ret;
1054	}
1055
1056	#define BYTES_TO_BITS(nb) ((BITS_PER_LONG * (nb)) / sizeof(long))
1057
1058	/* Bitfield type needs to be parsed into a fetch function */
1059	static int __parse_bitfield_probe_arg(const char *bf,
1060	const struct fetch_type *t,
1061	struct fetch_insn **pcode)
1062	{
1063	struct fetch_insn *code = *pcode;
1064	unsigned long bw, bo;
1065	char *tail;
1066
1067	if (*bf != 'b')
1068	return 0;
1069
1070	bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */
1071
1072	if (bw == 0 || *tail != '@')
1073	return -EINVAL;
1074
1075	bf = tail + 1;
1076	bo = simple_strtoul(bf, &tail, 0);
1077
1078	if (tail == bf || *tail != '/')
1079	return -EINVAL;
1080	code++;
1081	if (code->op != FETCH_OP_NOP)
1082	return -EINVAL;
1083	*pcode = code;
1084
1085	code->op = FETCH_OP_MOD_BF;
1086	code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
1087	code->rshift = BYTES_TO_BITS(t->size) - bw;
1088	code->basesize = t->size;
1089
1090	return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
1091	}
1092
1093	/* String length checking wrapper */
1094	static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
1095	struct probe_arg *parg,
1096	struct traceprobe_parse_context *ctx)
1097	{
1098	struct fetch_insn *code, *scode, *tmp = NULL;
1099	char *t, *t2, *t3;
1100	int ret, len;
1101	char *arg;
1102
1103	arg = kstrdup(s: argv, GFP_KERNEL);
1104	if (!arg)
1105	return -ENOMEM;
1106
1107	ret = -EINVAL;
1108	len = strlen(arg);
1109	if (len > MAX_ARGSTR_LEN) {
1110	trace_probe_log_err(ctx->offset, ARG_TOO_LONG);
1111	goto out;
1112	} else if (len == 0) {
1113	trace_probe_log_err(ctx->offset, NO_ARG_BODY);
1114	goto out;
1115	}
1116
1117	ret = -ENOMEM;
1118	parg->comm = kstrdup(s: arg, GFP_KERNEL);
1119	if (!parg->comm)
1120	goto out;
1121
1122	ret = -EINVAL;
1123	t = strchr(arg, ':');
1124	if (t) {
1125	*t = '\0';
1126	t2 = strchr(++t, '[');
1127	if (t2) {
1128	*t2++ = '\0';
1129	t3 = strchr(t2, ']');
1130	if (!t3) {
1131	int offs = t2 + strlen(t2) - arg;
1132
1133	trace_probe_log_err(ctx->offset + offs,
1134	ARRAY_NO_CLOSE);
1135	goto out;
1136	} else if (t3[1] != '\0') {
1137	trace_probe_log_err(ctx->offset + t3 + 1 - arg,
1138	BAD_ARRAY_SUFFIX);
1139	goto out;
1140	}
1141	*t3 = '\0';
1142	if (kstrtouint(s: t2, base: 0, res: &parg->count) || !parg->count) {
1143	trace_probe_log_err(ctx->offset + t2 - arg,
1144	BAD_ARRAY_NUM);
1145	goto out;
1146	}
1147	if (parg->count > MAX_ARRAY_LEN) {
1148	trace_probe_log_err(ctx->offset + t2 - arg,
1149	ARRAY_TOO_BIG);
1150	goto out;
1151	}
1152	}
1153	}
1154
1155	/*
1156	* Since $comm and immediate string can not be dereferenced,
1157	* we can find those by strcmp. But ignore for eprobes.
1158	*/
1159	if (!(ctx->flags & TPARG_FL_TEVENT) &&
1160	(strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
1161	strncmp(arg, "\\\"", 2) == 0)) {
1162	/* The type of $comm must be "string", and not an array. */
1163	if (parg->count || (t && strcmp(t, "string")))
1164	goto out;
1165	parg->type = find_fetch_type(type: "string", flags: ctx->flags);
1166	} else
1167	parg->type = find_fetch_type(type: t, flags: ctx->flags);
1168	if (!parg->type) {
1169	trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0), BAD_TYPE);
1170	goto out;
1171	}
1172	parg->offset = *size;
1173	*size += parg->type->size * (parg->count ?: 1);
1174
1175	ret = -ENOMEM;
1176	if (parg->count) {
1177	len = strlen(parg->type->fmttype) + 6;
1178	parg->fmt = kmalloc(size: len, GFP_KERNEL);
1179	if (!parg->fmt)
1180	goto out;
1181	snprintf(buf: parg->fmt, size: len, fmt: "%s[%d]", parg->type->fmttype,
1182	parg->count);
1183	}
1184
1185	code = tmp = kcalloc(FETCH_INSN_MAX, size: sizeof(*code), GFP_KERNEL);
1186	if (!code)
1187	goto out;
1188	code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
1189
1190	ctx->last_type = NULL;
1191	ret = parse_probe_arg(arg, type: parg->type, pcode: &code, end: &code[FETCH_INSN_MAX - 1],
1192	ctx);
1193	if (ret)
1194	goto fail;
1195
1196	/* Update storing type if BTF is available */
1197	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1198	ctx->last_type) {
1199	if (!t) {
1200	parg->type = find_fetch_type_from_btf_type(ctx);
1201	} else if (strstr(t, "string")) {
1202	ret = check_prepare_btf_string_fetch(typename: t, pcode: &code, ctx);
1203	if (ret)
1204	goto fail;
1205	}
1206	}
1207
1208	ret = -EINVAL;
1209	/* Store operation */
1210	if (parg->type->is_string) {
1211	if (!strcmp(parg->type->name, "symstr")) {
1212	if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
1213	code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
1214	code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
1215	trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1216	BAD_SYMSTRING);
1217	goto fail;
1218	}
1219	} else {
1220	if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
1221	code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
1222	code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
1223	trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1224	BAD_STRING);
1225	goto fail;
1226	}
1227	}
1228	if (!strcmp(parg->type->name, "symstr") ||
1229	(code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
1230	code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
1231	parg->count) {
1232	/*
1233	* IMM, DATA and COMM is pointing actual address, those
1234	* must be kept, and if parg->count != 0, this is an
1235	* array of string pointers instead of string address
1236	* itself.
1237	* For the symstr, it doesn't need to dereference, thus
1238	* it just get the value.
1239	*/
1240	code++;
1241	if (code->op != FETCH_OP_NOP) {
1242	trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1243	goto fail;
1244	}
1245	}
1246	/* If op == DEREF, replace it with STRING */
1247	if (!strcmp(parg->type->name, "ustring") ||
1248	code->op == FETCH_OP_UDEREF)
1249	code->op = FETCH_OP_ST_USTRING;
1250	else if (!strcmp(parg->type->name, "symstr"))
1251	code->op = FETCH_OP_ST_SYMSTR;
1252	else
1253	code->op = FETCH_OP_ST_STRING;
1254	code->size = parg->type->size;
1255	parg->dynamic = true;
1256	} else if (code->op == FETCH_OP_DEREF) {
1257	code->op = FETCH_OP_ST_MEM;
1258	code->size = parg->type->size;
1259	} else if (code->op == FETCH_OP_UDEREF) {
1260	code->op = FETCH_OP_ST_UMEM;
1261	code->size = parg->type->size;
1262	} else {
1263	code++;
1264	if (code->op != FETCH_OP_NOP) {
1265	trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1266	goto fail;
1267	}
1268	code->op = FETCH_OP_ST_RAW;
1269	code->size = parg->type->size;
1270	}
1271	scode = code;
1272	/* Modify operation */
1273	if (t != NULL) {
1274	ret = __parse_bitfield_probe_arg(bf: t, t: parg->type, pcode: &code);
1275	if (ret) {
1276	trace_probe_log_err(ctx->offset + t - arg, BAD_BITFIELD);
1277	goto fail;
1278	}
1279	} else if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1280	ctx->last_type) {
1281	ret = parse_btf_bitfield(pcode: &code, ctx);
1282	if (ret)
1283	goto fail;
1284	}
1285	ret = -EINVAL;
1286	/* Loop(Array) operation */
1287	if (parg->count) {
1288	if (scode->op != FETCH_OP_ST_MEM &&
1289	scode->op != FETCH_OP_ST_STRING &&
1290	scode->op != FETCH_OP_ST_USTRING) {
1291	trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1292	BAD_STRING);
1293	goto fail;
1294	}
1295	code++;
1296	if (code->op != FETCH_OP_NOP) {
1297	trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1298	goto fail;
1299	}
1300	code->op = FETCH_OP_LP_ARRAY;
1301	code->param = parg->count;
1302	}
1303	code++;
1304	code->op = FETCH_OP_END;
1305
1306	ret = 0;
1307	/* Shrink down the code buffer */
1308	parg->code = kcalloc(n: code - tmp + 1, size: sizeof(*code), GFP_KERNEL);
1309	if (!parg->code)
1310	ret = -ENOMEM;
1311	else
1312	memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
1313
1314	fail:
1315	if (ret) {
1316	for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
1317	if (code->op == FETCH_NOP_SYMBOL ||
1318	code->op == FETCH_OP_DATA)
1319	kfree(objp: code->data);
1320	}
1321	kfree(objp: tmp);
1322	out:
1323	kfree(objp: arg);
1324
1325	return ret;
1326	}
1327
1328	/* Return 1 if name is reserved or already used by another argument */
1329	static int traceprobe_conflict_field_name(const char *name,
1330	struct probe_arg *args, int narg)
1331	{
1332	int i;
1333
1334	for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
1335	if (strcmp(reserved_field_names[i], name) == 0)
1336	return 1;
1337
1338	for (i = 0; i < narg; i++)
1339	if (strcmp(args[i].name, name) == 0)
1340	return 1;
1341
1342	return 0;
1343	}
1344
1345	static char *generate_probe_arg_name(const char *arg, int idx)
1346	{
1347	char *name = NULL;
1348	const char *end;
1349
1350	/*
1351	* If argument name is omitted, try arg as a name (BTF variable)
1352	* or "argN".
1353	*/
1354	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
1355	end = strchr(arg, ':');
1356	if (!end)
1357	end = arg + strlen(arg);
1358
1359	name = kmemdup_nul(s: arg, len: end - arg, GFP_KERNEL);
1360	if (!name || !is_good_name(name)) {
1361	kfree(objp: name);
1362	name = NULL;
1363	}
1364	}
1365
1366	if (!name)
1367	name = kasprintf(GFP_KERNEL, fmt: "arg%d", idx + 1);
1368
1369	return name;
1370	}
1371
1372	int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
1373	struct traceprobe_parse_context *ctx)
1374	{
1375	struct probe_arg *parg = &tp->args[i];
1376	const char *body;
1377
1378	/* Increment count for freeing args in error case */
1379	tp->nr_args++;
1380
1381	body = strchr(arg, '=');
1382	if (body) {
1383	if (body - arg > MAX_ARG_NAME_LEN) {
1384	trace_probe_log_err(0, ARG_NAME_TOO_LONG);
1385	return -EINVAL;
1386	} else if (body == arg) {
1387	trace_probe_log_err(0, NO_ARG_NAME);
1388	return -EINVAL;
1389	}
1390	parg->name = kmemdup_nul(s: arg, len: body - arg, GFP_KERNEL);
1391	body++;
1392	} else {
1393	parg->name = generate_probe_arg_name(arg, idx: i);
1394	body = arg;
1395	}
1396	if (!parg->name)
1397	return -ENOMEM;
1398
1399	if (!is_good_name(name: parg->name)) {
1400	trace_probe_log_err(0, BAD_ARG_NAME);
1401	return -EINVAL;
1402	}
1403	if (traceprobe_conflict_field_name(name: parg->name, args: tp->args, narg: i)) {
1404	trace_probe_log_err(0, USED_ARG_NAME);
1405	return -EINVAL;
1406	}
1407	ctx->offset = body - arg;
1408	/* Parse fetch argument */
1409	return traceprobe_parse_probe_arg_body(argv: body, size: &tp->size, parg, ctx);
1410	}
1411
1412	void traceprobe_free_probe_arg(struct probe_arg *arg)
1413	{
1414	struct fetch_insn *code = arg->code;
1415
1416	while (code && code->op != FETCH_OP_END) {
1417	if (code->op == FETCH_NOP_SYMBOL ||
1418	code->op == FETCH_OP_DATA)
1419	kfree(objp: code->data);
1420	code++;
1421	}
1422	kfree(objp: arg->code);
1423	kfree(objp: arg->name);
1424	kfree(objp: arg->comm);
1425	kfree(objp: arg->fmt);
1426	}
1427
1428	static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
1429	struct traceprobe_parse_context *ctx)
1430	{
1431	int i, found = 0;
1432
1433	for (i = 0; i < argc; i++)
1434	if (str_has_prefix(str: argv[i], prefix: "$arg")) {
1435	trace_probe_log_set_index(index: i + 2);
1436
1437	if (!tparg_is_function_entry(flags: ctx->flags)) {
1438	trace_probe_log_err(0, NOFENTRY_ARGS);
1439	return -EINVAL;
1440	}
1441
1442	if (isdigit(c: argv[i][4])) {
1443	found = 1;
1444	continue;
1445	}
1446
1447	if (argv[i][4] != '*') {
1448	trace_probe_log_err(0, BAD_VAR);
1449	return -EINVAL;
1450	}
1451
1452	if (*args_idx >= 0 && *args_idx < argc) {
1453	trace_probe_log_err(0, DOUBLE_ARGS);
1454	return -EINVAL;
1455	}
1456	found = 1;
1457	*args_idx = i;
1458	}
1459
1460	return found;
1461	}
1462
1463	static int sprint_nth_btf_arg(int idx, const char *type,
1464	char *buf, int bufsize,
1465	struct traceprobe_parse_context *ctx)
1466	{
1467	const char *name;
1468	int ret;
1469
1470	if (idx >= ctx->nr_params) {
1471	trace_probe_log_err(0, NO_BTFARG);
1472	return -ENOENT;
1473	}
1474	name = btf_name_by_offset(btf: ctx->btf, offset: ctx->params[idx].name_off);
1475	if (!name) {
1476	trace_probe_log_err(0, NO_BTF_ENTRY);
1477	return -ENOENT;
1478	}
1479	ret = snprintf(buf, size: bufsize, fmt: "%s%s", name, type);
1480	if (ret >= bufsize) {
1481	trace_probe_log_err(0, ARGS_2LONG);
1482	return -E2BIG;
1483	}
1484	return ret;
1485	}
1486
1487	/* Return new_argv which must be freed after use */
1488	const char **traceprobe_expand_meta_args(int argc, const char *argv[],
1489	int *new_argc, char *buf, int bufsize,
1490	struct traceprobe_parse_context *ctx)
1491	{
1492	const struct btf_param *params = NULL;
1493	int i, j, n, used, ret, args_idx = -1;
1494	const char **new_argv = NULL;
1495
1496	ret = argv_has_var_arg(argc, argv, args_idx: &args_idx, ctx);
1497	if (ret < 0)
1498	return ERR_PTR(error: ret);
1499
1500	if (!ret) {
1501	*new_argc = argc;
1502	return NULL;
1503	}
1504
1505	ret = query_btf_context(ctx);
1506	if (ret < 0 || ctx->nr_params == 0) {
1507	if (args_idx != -1) {
1508	/* $arg* requires BTF info */
1509	trace_probe_log_err(0, NOSUP_BTFARG);
1510	return (const char **)params;
1511	}
1512	*new_argc = argc;
1513	return NULL;
1514	}
1515
1516	if (args_idx >= 0)
1517	*new_argc = argc + ctx->nr_params - 1;
1518	else
1519	*new_argc = argc;
1520
1521	new_argv = kcalloc(n: *new_argc, size: sizeof(char *), GFP_KERNEL);
1522	if (!new_argv)
1523	return ERR_PTR(error: -ENOMEM);
1524
1525	used = 0;
1526	for (i = 0, j = 0; i < argc; i++) {
1527	trace_probe_log_set_index(index: i + 2);
1528	if (i == args_idx) {
1529	for (n = 0; n < ctx->nr_params; n++) {
1530	ret = sprint_nth_btf_arg(idx: n, type: "", buf: buf + used,
1531	bufsize: bufsize - used, ctx);
1532	if (ret < 0)
1533	goto error;
1534
1535	new_argv[j++] = buf + used;
1536	used += ret + 1;
1537	}
1538	continue;
1539	}
1540
1541	if (str_has_prefix(str: argv[i], prefix: "$arg")) {
1542	char *type = NULL;
1543
1544	n = simple_strtoul(argv[i] + 4, &type, 10);
1545	if (type && !(*type == ':' || *type == '\0')) {
1546	trace_probe_log_err(0, BAD_VAR);
1547	ret = -ENOENT;
1548	goto error;
1549	}
1550	/* Note: $argN starts from $arg1 */
1551	ret = sprint_nth_btf_arg(idx: n - 1, type, buf: buf + used,
1552	bufsize: bufsize - used, ctx);
1553	if (ret < 0)
1554	goto error;
1555	new_argv[j++] = buf + used;
1556	used += ret + 1;
1557	} else
1558	new_argv[j++] = argv[i];
1559	}
1560
1561	return new_argv;
1562
1563	error:
1564	kfree(objp: new_argv);
1565	return ERR_PTR(error: ret);
1566	}
1567
1568	void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)
1569	{
1570	clear_btf_context(ctx);
1571	}
1572
1573	int traceprobe_update_arg(struct probe_arg *arg)
1574	{
1575	struct fetch_insn *code = arg->code;
1576	long offset;
1577	char *tmp;
1578	char c;
1579	int ret = 0;
1580
1581	while (code && code->op != FETCH_OP_END) {
1582	if (code->op == FETCH_NOP_SYMBOL) {
1583	if (code[1].op != FETCH_OP_IMM)
1584	return -EINVAL;
1585
1586	tmp = strpbrk(code->data, "+-");
1587	if (tmp)
1588	c = *tmp;
1589	ret = traceprobe_split_symbol_offset(symbol: code->data,
1590	offset: &offset);
1591	if (ret)
1592	return ret;
1593
1594	code[1].immediate =
1595	(unsigned long)kallsyms_lookup_name(name: code->data);
1596	if (tmp)
1597	*tmp = c;
1598	if (!code[1].immediate)
1599	return -ENOENT;
1600	code[1].immediate += offset;
1601	}
1602	code++;
1603	}
1604	return 0;
1605	}
1606
1607	/* When len=0, we just calculate the needed length */
1608	#define LEN_OR_ZERO (len ? len - pos : 0)
1609	static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
1610	enum probe_print_type ptype)
1611	{
1612	struct probe_arg *parg;
1613	int i, j;
1614	int pos = 0;
1615	const char *fmt, *arg;
1616
1617	switch (ptype) {
1618	case PROBE_PRINT_NORMAL:
1619	fmt = "(%lx)";
1620	arg = ", REC->" FIELD_STRING_IP;
1621	break;
1622	case PROBE_PRINT_RETURN:
1623	fmt = "(%lx <- %lx)";
1624	arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1625	break;
1626	case PROBE_PRINT_EVENT:
1627	fmt = "";
1628	arg = "";
1629	break;
1630	default:
1631	WARN_ON_ONCE(1);
1632	return 0;
1633	}
1634
1635	pos += snprintf(buf: buf + pos, LEN_OR_ZERO, fmt: "\"%s", fmt);
1636
1637	for (i = 0; i < tp->nr_args; i++) {
1638	parg = tp->args + i;
1639	pos += snprintf(buf: buf + pos, LEN_OR_ZERO, fmt: " %s=", parg->name);
1640	if (parg->count) {
1641	pos += snprintf(buf: buf + pos, LEN_OR_ZERO, fmt: "{%s",
1642	parg->type->fmt);
1643	for (j = 1; j < parg->count; j++)
1644	pos += snprintf(buf: buf + pos, LEN_OR_ZERO, fmt: ",%s",
1645	parg->type->fmt);
1646	pos += snprintf(buf: buf + pos, LEN_OR_ZERO, fmt: "}");
1647	} else
1648	pos += snprintf(buf: buf + pos, LEN_OR_ZERO, fmt: "%s",
1649	parg->type->fmt);
1650	}
1651
1652	pos += snprintf(buf: buf + pos, LEN_OR_ZERO, fmt: "\"%s", arg);
1653
1654	for (i = 0; i < tp->nr_args; i++) {
1655	parg = tp->args + i;
1656	if (parg->count) {
1657	if (parg->type->is_string)
1658	fmt = ", __get_str(%s[%d])";
1659	else
1660	fmt = ", REC->%s[%d]";
1661	for (j = 0; j < parg->count; j++)
1662	pos += snprintf(buf: buf + pos, LEN_OR_ZERO,
1663	fmt, parg->name, j);
1664	} else {
1665	if (parg->type->is_string)
1666	fmt = ", __get_str(%s)";
1667	else
1668	fmt = ", REC->%s";
1669	pos += snprintf(buf: buf + pos, LEN_OR_ZERO,
1670	fmt, parg->name);
1671	}
1672	}
1673
1674	/* return the length of print_fmt */
1675	return pos;
1676	}
1677	#undef LEN_OR_ZERO
1678
1679	int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
1680	{
1681	struct trace_event_call *call = trace_probe_event_call(tp);
1682	int len;
1683	char *print_fmt;
1684
1685	/* First: called with 0 length to calculate the needed length */
1686	len = __set_print_fmt(tp, NULL, len: 0, ptype);
1687	print_fmt = kmalloc(size: len + 1, GFP_KERNEL);
1688	if (!print_fmt)
1689	return -ENOMEM;
1690
1691	/* Second: actually write the @print_fmt */
1692	__set_print_fmt(tp, buf: print_fmt, len: len + 1, ptype);
1693	call->print_fmt = print_fmt;
1694
1695	return 0;
1696	}
1697
1698	int traceprobe_define_arg_fields(struct trace_event_call *event_call,
1699	size_t offset, struct trace_probe *tp)
1700	{
1701	int ret, i;
1702
1703	/* Set argument names as fields */
1704	for (i = 0; i < tp->nr_args; i++) {
1705	struct probe_arg *parg = &tp->args[i];
1706	const char *fmt = parg->type->fmttype;
1707	int size = parg->type->size;
1708
1709	if (parg->fmt)
1710	fmt = parg->fmt;
1711	if (parg->count)
1712	size *= parg->count;
1713	ret = trace_define_field(call: event_call, type: fmt, name: parg->name,
1714	offset: offset + parg->offset, size,
1715	is_signed: parg->type->is_signed,
1716	filter_type: FILTER_OTHER);
1717	if (ret)
1718	return ret;
1719	}
1720	return 0;
1721	}
1722
1723	static void trace_probe_event_free(struct trace_probe_event *tpe)
1724	{
1725	kfree(objp: tpe->class.system);
1726	kfree(objp: tpe->call.name);
1727	kfree(objp: tpe->call.print_fmt);
1728	kfree(objp: tpe);
1729	}
1730
1731	int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
1732	{
1733	if (trace_probe_has_sibling(tp))
1734	return -EBUSY;
1735
1736	list_del_init(entry: &tp->list);
1737	trace_probe_event_free(tpe: tp->event);
1738
1739	tp->event = to->event;
1740	list_add_tail(new: &tp->list, head: trace_probe_probe_list(tp: to));
1741
1742	return 0;
1743	}
1744
1745	void trace_probe_unlink(struct trace_probe *tp)
1746	{
1747	list_del_init(entry: &tp->list);
1748	if (list_empty(head: trace_probe_probe_list(tp)))
1749	trace_probe_event_free(tpe: tp->event);
1750	tp->event = NULL;
1751	}
1752
1753	void trace_probe_cleanup(struct trace_probe *tp)
1754	{
1755	int i;
1756
1757	for (i = 0; i < tp->nr_args; i++)
1758	traceprobe_free_probe_arg(arg: &tp->args[i]);
1759
1760	if (tp->event)
1761	trace_probe_unlink(tp);
1762	}
1763
1764	int trace_probe_init(struct trace_probe *tp, const char *event,
1765	const char *group, bool alloc_filter)
1766	{
1767	struct trace_event_call *call;
1768	size_t size = sizeof(struct trace_probe_event);
1769	int ret = 0;
1770
1771	if (!event || !group)
1772	return -EINVAL;
1773
1774	if (alloc_filter)
1775	size += sizeof(struct trace_uprobe_filter);
1776
1777	tp->event = kzalloc(size, GFP_KERNEL);
1778	if (!tp->event)
1779	return -ENOMEM;
1780
1781	INIT_LIST_HEAD(list: &tp->event->files);
1782	INIT_LIST_HEAD(list: &tp->event->class.fields);
1783	INIT_LIST_HEAD(list: &tp->event->probes);
1784	INIT_LIST_HEAD(list: &tp->list);
1785	list_add(new: &tp->list, head: &tp->event->probes);
1786
1787	call = trace_probe_event_call(tp);
1788	call->class = &tp->event->class;
1789	call->name = kstrdup(s: event, GFP_KERNEL);
1790	if (!call->name) {
1791	ret = -ENOMEM;
1792	goto error;
1793	}
1794
1795	tp->event->class.system = kstrdup(s: group, GFP_KERNEL);
1796	if (!tp->event->class.system) {
1797	ret = -ENOMEM;
1798	goto error;
1799	}
1800
1801	return 0;
1802
1803	error:
1804	trace_probe_cleanup(tp);
1805	return ret;
1806	}
1807
1808	static struct trace_event_call *
1809	find_trace_event_call(const char *system, const char *event_name)
1810	{
1811	struct trace_event_call *tp_event;
1812	const char *name;
1813
1814	list_for_each_entry(tp_event, &ftrace_events, list) {
1815	if (!tp_event->class->system ||
1816	strcmp(system, tp_event->class->system))
1817	continue;
1818	name = trace_event_name(call: tp_event);
1819	if (!name || strcmp(event_name, name))
1820	continue;
1821	return tp_event;
1822	}
1823
1824	return NULL;
1825	}
1826
1827	int trace_probe_register_event_call(struct trace_probe *tp)
1828	{
1829	struct trace_event_call *call = trace_probe_event_call(tp);
1830	int ret;
1831
1832	lockdep_assert_held(&event_mutex);
1833
1834	if (find_trace_event_call(system: trace_probe_group_name(tp),
1835	event_name: trace_probe_name(tp)))
1836	return -EEXIST;
1837
1838	ret = register_trace_event(event: &call->event);
1839	if (!ret)
1840	return -ENODEV;
1841
1842	ret = trace_add_event_call(call);
1843	if (ret)
1844	unregister_trace_event(event: &call->event);
1845
1846	return ret;
1847	}
1848
1849	int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
1850	{
1851	struct event_file_link *link;
1852
1853	link = kmalloc(size: sizeof(*link), GFP_KERNEL);
1854	if (!link)
1855	return -ENOMEM;
1856
1857	link->file = file;
1858	INIT_LIST_HEAD(list: &link->list);
1859	list_add_tail_rcu(new: &link->list, head: &tp->event->files);
1860	trace_probe_set_flag(tp, TP_FLAG_TRACE);
1861	return 0;
1862	}
1863
1864	struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
1865	struct trace_event_file *file)
1866	{
1867	struct event_file_link *link;
1868
1869	trace_probe_for_each_link(link, tp) {
1870	if (link->file == file)
1871	return link;
1872	}
1873
1874	return NULL;
1875	}
1876
1877	int trace_probe_remove_file(struct trace_probe *tp,
1878	struct trace_event_file *file)
1879	{
1880	struct event_file_link *link;
1881
1882	link = trace_probe_get_file_link(tp, file);
1883	if (!link)
1884	return -ENOENT;
1885
1886	list_del_rcu(entry: &link->list);
1887	kvfree_rcu_mightsleep(link);
1888
1889	if (list_empty(head: &tp->event->files))
1890	trace_probe_clear_flag(tp, TP_FLAG_TRACE);
1891
1892	return 0;
1893	}
1894
1895	/*
1896	* Return the smallest index of different type argument (start from 1).
1897	* If all argument types and name are same, return 0.
1898	*/
1899	int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
1900	{
1901	int i;
1902
1903	/* In case of more arguments */
1904	if (a->nr_args < b->nr_args)
1905	return a->nr_args + 1;
1906	if (a->nr_args > b->nr_args)
1907	return b->nr_args + 1;
1908
1909	for (i = 0; i < a->nr_args; i++) {
1910	if ((b->nr_args <= i) ||
1911	((a->args[i].type != b->args[i].type) ||
1912	(a->args[i].count != b->args[i].count) ||
1913	strcmp(a->args[i].name, b->args[i].name)))
1914	return i + 1;
1915	}
1916
1917	return 0;
1918	}
1919
1920	bool trace_probe_match_command_args(struct trace_probe *tp,
1921	int argc, const char **argv)
1922	{
1923	char buf[MAX_ARGSTR_LEN + 1];
1924	int i;
1925
1926	if (tp->nr_args < argc)
1927	return false;
1928
1929	for (i = 0; i < argc; i++) {
1930	snprintf(buf, size: sizeof(buf), fmt: "%s=%s",
1931	tp->args[i].name, tp->args[i].comm);
1932	if (strcmp(buf, argv[i]))
1933	return false;
1934	}
1935	return true;
1936	}
1937
1938	int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
1939	{
1940	int argc = 0, ret = 0;
1941	char **argv;
1942
1943	argv = argv_split(GFP_KERNEL, str: raw_command, argcp: &argc);
1944	if (!argv)
1945	return -ENOMEM;
1946
1947	if (argc)
1948	ret = createfn(argc, (const char **)argv);
1949
1950	argv_free(argv);
1951
1952	return ret;
1953	}
1954
1955	int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
1956	u8 *data, void *field)
1957	{
1958	void *p;
1959	int i, j;
1960
1961	for (i = 0; i < nr_args; i++) {
1962	struct probe_arg *a = args + i;
1963
1964	trace_seq_printf(s, fmt: " %s=", a->name);
1965	if (likely(!a->count)) {
1966	if (!a->type->print(s, data + a->offset, field))
1967	return -ENOMEM;
1968	continue;
1969	}
1970	trace_seq_putc(s, c: '{');
1971	p = data + a->offset;
1972	for (j = 0; j < a->count; j++) {
1973	if (!a->type->print(s, p, field))
1974	return -ENOMEM;
1975	trace_seq_putc(s, c: j == a->count - 1 ? '}' : ',');
1976	p += a->type->size;
1977	}
1978	}
1979	return 0;
1980	}
1981