1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* arch/arm/kernel/kprobes-test.c
4	*
5	* Copyright (C) 2011 Jon Medhurst <tixy@yxit.co.uk>.
6	*/
7
8	/*
9	* This file contains test code for ARM kprobes.
10	*
11	* The top level function run_all_tests() executes tests for all of the
12	* supported instruction sets: ARM, 16-bit Thumb, and 32-bit Thumb. These tests
13	* fall into two categories; run_api_tests() checks basic functionality of the
14	* kprobes API, and run_test_cases() is a comprehensive test for kprobes
15	* instruction decoding and simulation.
16	*
17	* run_test_cases() first checks the kprobes decoding table for self consistency
18	* (using table_test()) then executes a series of test cases for each of the CPU
19	* instruction forms. coverage_start() and coverage_end() are used to verify
20	* that these test cases cover all of the possible combinations of instructions
21	* described by the kprobes decoding tables.
22	*
23	* The individual test cases are in kprobes-test-arm.c and kprobes-test-thumb.c
24	* which use the macros defined in kprobes-test.h. The rest of this
25	* documentation will describe the operation of the framework used by these
26	* test cases.
27	*/
28
29	/*
30	* TESTING METHODOLOGY
31	* -------------------
32	*
33	* The methodology used to test an ARM instruction 'test_insn' is to use
34	* inline assembler like:
35	*
36	* test_before: nop
37	* test_case: test_insn
38	* test_after: nop
39	*
40	* When the test case is run a kprobe is placed of each nop. The
41	* post-handler of the test_before probe is used to modify the saved CPU
42	* register context to that which we require for the test case. The
43	* pre-handler of the of the test_after probe saves a copy of the CPU
44	* register context. In this way we can execute test_insn with a specific
45	* register context and see the results afterwards.
46	*
47	* To actually test the kprobes instruction emulation we perform the above
48	* step a second time but with an additional kprobe on the test_case
49	* instruction itself. If the emulation is accurate then the results seen
50	* by the test_after probe will be identical to the first run which didn't
51	* have a probe on test_case.
52	*
53	* Each test case is run several times with a variety of variations in the
54	* flags value of stored in CPSR, and for Thumb code, different ITState.
55	*
56	* For instructions which can modify PC, a second test_after probe is used
57	* like this:
58	*
59	* test_before: nop
60	* test_case: test_insn
61	* test_after: nop
62	* b test_done
63	* test_after2: nop
64	* test_done:
65	*
66	* The test case is constructed such that test_insn branches to
67	* test_after2, or, if testing a conditional instruction, it may just
68	* continue to test_after. The probes inserted at both locations let us
69	* determine which happened. A similar approach is used for testing
70	* backwards branches...
71	*
72	* b test_before
73	* b test_done @ helps to cope with off by 1 branches
74	* test_after2: nop
75	* b test_done
76	* test_before: nop
77	* test_case: test_insn
78	* test_after: nop
79	* test_done:
80	*
81	* The macros used to generate the assembler instructions describe above
82	* are TEST_INSTRUCTION, TEST_BRANCH_F (branch forwards) and TEST_BRANCH_B
83	* (branch backwards). In these, the local variables numbered 1, 50, 2 and
84	* 99 represent: test_before, test_case, test_after2 and test_done.
85	*
86	* FRAMEWORK
87	* ---------
88	*
89	* Each test case is wrapped between the pair of macros TESTCASE_START and
90	* TESTCASE_END. As well as performing the inline assembler boilerplate,
91	* these call out to the kprobes_test_case_start() and
92	* kprobes_test_case_end() functions which drive the execution of the test
93	* case. The specific arguments to use for each test case are stored as
94	* inline data constructed using the various TEST_ARG_* macros. Putting
95	* this all together, a simple test case may look like:
96	*
97	* TESTCASE_START("Testing mov r0, r7")
98	* TEST_ARG_REG(7, 0x12345678) // Set r7=0x12345678
99	* TEST_ARG_END("")
100	* TEST_INSTRUCTION("mov r0, r7")
101	* TESTCASE_END
102	*
103	* Note, in practice the single convenience macro TEST_R would be used for this
104	* instead.
105	*
106	* The above would expand to assembler looking something like:
107	*
108	* @ TESTCASE_START
109	* bl __kprobes_test_case_start
110	* .pushsection .rodata
111	* "10:
112	* .ascii "mov r0, r7" @ text title for test case
113	* .byte 0
114	* .popsection
115	* @ start of inline data...
116	* .word 10b @ pointer to title in .rodata section
117	*
118	* @ TEST_ARG_REG
119	* .byte ARG_TYPE_REG
120	* .byte 7
121	* .short 0
122	* .word 0x1234567
123	*
124	* @ TEST_ARG_END
125	* .byte ARG_TYPE_END
126	* .byte TEST_ISA @ flags, including ISA being tested
127	* .short 50f-0f @ offset of 'test_before'
128	* .short 2f-0f @ offset of 'test_after2' (if relevent)
129	* .short 99f-0f @ offset of 'test_done'
130	* @ start of test case code...
131	* 0:
132	* .code TEST_ISA @ switch to ISA being tested
133	*
134	* @ TEST_INSTRUCTION
135	* 50: nop @ location for 'test_before' probe
136	* 1: mov r0, r7 @ the test case instruction 'test_insn'
137	* nop @ location for 'test_after' probe
138	*
139	* // TESTCASE_END
140	* 2:
141	* 99: bl __kprobes_test_case_end_##TEST_ISA
142	* .code NONMAL_ISA
143	*
144	* When the above is execute the following happens...
145	*
146	* __kprobes_test_case_start() is an assembler wrapper which sets up space
147	* for a stack buffer and calls the C function kprobes_test_case_start().
148	* This C function will do some initial processing of the inline data and
149	* setup some global state. It then inserts the test_before and test_after
150	* kprobes and returns a value which causes the assembler wrapper to jump
151	* to the start of the test case code, (local label '0').
152	*
153	* When the test case code executes, the test_before probe will be hit and
154	* test_before_post_handler will call setup_test_context(). This fills the
155	* stack buffer and CPU registers with a test pattern and then processes
156	* the test case arguments. In our example there is one TEST_ARG_REG which
157	* indicates that R7 should be loaded with the value 0x12345678.
158	*
159	* When the test_before probe ends, the test case continues and executes
160	* the "mov r0, r7" instruction. It then hits the test_after probe and the
161	* pre-handler for this (test_after_pre_handler) will save a copy of the
162	* CPU register context. This should now have R0 holding the same value as
163	* R7.
164	*
165	* Finally we get to the call to __kprobes_test_case_end_{32,16}. This is
166	* an assembler wrapper which switches back to the ISA used by the test
167	* code and calls the C function kprobes_test_case_end().
168	*
169	* For each run through the test case, test_case_run_count is incremented
170	* by one. For even runs, kprobes_test_case_end() saves a copy of the
171	* register and stack buffer contents from the test case just run. It then
172	* inserts a kprobe on the test case instruction 'test_insn' and returns a
173	* value to cause the test case code to be re-run.
174	*
175	* For odd numbered runs, kprobes_test_case_end() compares the register and
176	* stack buffer contents to those that were saved on the previous even
177	* numbered run (the one without the kprobe on test_insn). These should be
178	* the same if the kprobe instruction simulation routine is correct.
179	*
180	* The pair of test case runs is repeated with different combinations of
181	* flag values in CPSR and, for Thumb, different ITState. This is
182	* controlled by test_context_cpsr().
183	*
184	* BUILDING TEST CASES
185	* -------------------
186	*
187	*
188	* As an aid to building test cases, the stack buffer is initialised with
189	* some special values:
190	*
191	* [SP+13*4] Contains SP+120. This can be used to test instructions
192	* which load a value into SP.
193	*
194	* [SP+15*4] When testing branching instructions using TEST_BRANCH_{F,B},
195	* this holds the target address of the branch, 'test_after2'.
196	* This can be used to test instructions which load a PC value
197	* from memory.
198	*/
199
200	#include <linux/kernel.h>
201	#include <linux/module.h>
202	#include <linux/slab.h>
203	#include <linux/sched/clock.h>
204	#include <linux/kprobes.h>
205	#include <linux/errno.h>
206	#include <linux/stddef.h>
207	#include <linux/bug.h>
208	#include <asm/opcodes.h>
209
210	#include "core.h"
211	#include "test-core.h"
212	#include "../decode-arm.h"
213	#include "../decode-thumb.h"
214
215
216	#define BENCHMARKING 1
217
218
219	/*
220	* Test basic API
221	*/
222
223	static bool test_regs_ok;
224	static int test_func_instance;
225	static int pre_handler_called;
226	static int post_handler_called;
227	static int kretprobe_handler_called;
228	static int tests_failed;
229
230	#define FUNC_ARG1 0x12345678
231	#define FUNC_ARG2 0xabcdef
232
233
234	#ifndef CONFIG_THUMB2_KERNEL
235
236	#define RET(reg) "mov pc, "#reg
237
238	long arm_func(long r0, long r1);
239
240	static void __used __naked __arm_kprobes_test_func(void)
241	{
242	__asm__ __volatile__ (
243	".arm \n\t"
244	".type arm_func, %%function \n\t"
245	"arm_func: \n\t"
246	"adds r0, r0, r1 \n\t"
247	"mov pc, lr \n\t"
248	".code "NORMAL_ISA /* Back to Thumb if necessary */
249	: : : "r0", "r1", "cc"
250	);
251	}
252
253	#else /* CONFIG_THUMB2_KERNEL */
254
255	#define RET(reg) "bx "#reg
256
257	long thumb16_func(long r0, long r1);
258	long thumb32even_func(long r0, long r1);
259	long thumb32odd_func(long r0, long r1);
260
261	static void __used __naked __thumb_kprobes_test_funcs(void)
262	{
263	__asm__ __volatile__ (
264	".type thumb16_func, %%function \n\t"
265	"thumb16_func: \n\t"
266	"adds.n r0, r0, r1 \n\t"
267	"bx lr \n\t"
268
269	".align \n\t"
270	".type thumb32even_func, %%function \n\t"
271	"thumb32even_func: \n\t"
272	"adds.w r0, r0, r1 \n\t"
273	"bx lr \n\t"
274
275	".align \n\t"
276	"nop.n \n\t"
277	".type thumb32odd_func, %%function \n\t"
278	"thumb32odd_func: \n\t"
279	"adds.w r0, r0, r1 \n\t"
280	"bx lr \n\t"
281
282	: : : "r0", "r1", "cc"
283	);
284	}
285
286	#endif /* CONFIG_THUMB2_KERNEL */
287
288
289	static int call_test_func(long (*func)(long, long), bool check_test_regs)
290	{
291	long ret;
292
293	++test_func_instance;
294	test_regs_ok = false;
295
296	ret = (*func)(FUNC_ARG1, FUNC_ARG2);
297	if (ret != FUNC_ARG1 + FUNC_ARG2) {
298	pr_err("FAIL: call_test_func: func returned %lx\n", ret);
299	return false;
300	}
301
302	if (check_test_regs && !test_regs_ok) {
303	pr_err("FAIL: test regs not OK\n");
304	return false;
305	}
306
307	return true;
308	}
309
310	static int __kprobes pre_handler(struct kprobe *p, struct pt_regs *regs)
311	{
312	pre_handler_called = test_func_instance;
313	if (regs->ARM_r0 == FUNC_ARG1 && regs->ARM_r1 == FUNC_ARG2)
314	test_regs_ok = true;
315	return 0;
316	}
317
318	static void __kprobes post_handler(struct kprobe *p, struct pt_regs *regs,
319	unsigned long flags)
320	{
321	post_handler_called = test_func_instance;
322	if (regs->ARM_r0 != FUNC_ARG1 + FUNC_ARG2 || regs->ARM_r1 != FUNC_ARG2)
323	test_regs_ok = false;
324	}
325
326	static struct kprobe the_kprobe = {
327	.addr = 0,
328	.pre_handler = pre_handler,
329	.post_handler = post_handler
330	};
331
332	static int test_kprobe(long (*func)(long, long))
333	{
334	int ret;
335
336	the_kprobe.addr = (kprobe_opcode_t *)func;
337	ret = register_kprobe(p: &the_kprobe);
338	if (ret < 0) {
339	pr_err("FAIL: register_kprobe failed with %d\n", ret);
340	return ret;
341	}
342
343	ret = call_test_func(func, check_test_regs: true);
344
345	unregister_kprobe(p: &the_kprobe);
346	the_kprobe.flags = 0; /* Clear disable flag to allow reuse */
347
348	if (!ret)
349	return -EINVAL;
350	if (pre_handler_called != test_func_instance) {
351	pr_err("FAIL: kprobe pre_handler not called\n");
352	return -EINVAL;
353	}
354	if (post_handler_called != test_func_instance) {
355	pr_err("FAIL: kprobe post_handler not called\n");
356	return -EINVAL;
357	}
358	if (!call_test_func(func, check_test_regs: false))
359	return -EINVAL;
360	if (pre_handler_called == test_func_instance ||
361	post_handler_called == test_func_instance) {
362	pr_err("FAIL: probe called after unregistering\n");
363	return -EINVAL;
364	}
365
366	return 0;
367	}
368
369	static int __kprobes
370	kretprobe_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
371	{
372	kretprobe_handler_called = test_func_instance;
373	if (regs_return_value(regs) == FUNC_ARG1 + FUNC_ARG2)
374	test_regs_ok = true;
375	return 0;
376	}
377
378	static struct kretprobe the_kretprobe = {
379	.handler = kretprobe_handler,
380	};
381
382	static int test_kretprobe(long (*func)(long, long))
383	{
384	int ret;
385
386	the_kretprobe.kp.addr = (kprobe_opcode_t *)func;
387	ret = register_kretprobe(rp: &the_kretprobe);
388	if (ret < 0) {
389	pr_err("FAIL: register_kretprobe failed with %d\n", ret);
390	return ret;
391	}
392
393	ret = call_test_func(func, check_test_regs: true);
394
395	unregister_kretprobe(rp: &the_kretprobe);
396	the_kretprobe.kp.flags = 0; /* Clear disable flag to allow reuse */
397
398	if (!ret)
399	return -EINVAL;
400	if (kretprobe_handler_called != test_func_instance) {
401	pr_err("FAIL: kretprobe handler not called\n");
402	return -EINVAL;
403	}
404	if (!call_test_func(func, check_test_regs: false))
405	return -EINVAL;
406	if (kretprobe_handler_called == test_func_instance) {
407	pr_err("FAIL: kretprobe called after unregistering\n");
408	return -EINVAL;
409	}
410
411	return 0;
412	}
413
414	static int run_api_tests(long (*func)(long, long))
415	{
416	int ret;
417
418	pr_info(" kprobe\n");
419	ret = test_kprobe(func);
420	if (ret < 0)
421	return ret;
422
423	pr_info(" kretprobe\n");
424	ret = test_kretprobe(func);
425	if (ret < 0)
426	return ret;
427
428	return 0;
429	}
430
431
432	/*
433	* Benchmarking
434	*/
435
436	#if BENCHMARKING
437
438	static void __naked benchmark_nop(void)
439	{
440	__asm__ __volatile__ (
441	"nop \n\t"
442	RET(lr)" \n\t"
443	);
444	}
445
446	#ifdef CONFIG_THUMB2_KERNEL
447	#define wide ".w"
448	#else
449	#define wide
450	#endif
451
452	static void __naked benchmark_pushpop1(void)
453	{
454	__asm__ __volatile__ (
455	"stmdb"wide" sp!, {r3-r11,lr} \n\t"
456	"ldmia"wide" sp!, {r3-r11,pc}"
457	);
458	}
459
460	static void __naked benchmark_pushpop2(void)
461	{
462	__asm__ __volatile__ (
463	"stmdb"wide" sp!, {r0-r8,lr} \n\t"
464	"ldmia"wide" sp!, {r0-r8,pc}"
465	);
466	}
467
468	static void __naked benchmark_pushpop3(void)
469	{
470	__asm__ __volatile__ (
471	"stmdb"wide" sp!, {r4,lr} \n\t"
472	"ldmia"wide" sp!, {r4,pc}"
473	);
474	}
475
476	static void __naked benchmark_pushpop4(void)
477	{
478	__asm__ __volatile__ (
479	"stmdb"wide" sp!, {r0,lr} \n\t"
480	"ldmia"wide" sp!, {r0,pc}"
481	);
482	}
483
484
485	#ifdef CONFIG_THUMB2_KERNEL
486
487	static void __naked benchmark_pushpop_thumb(void)
488	{
489	__asm__ __volatile__ (
490	"push.n {r0-r7,lr} \n\t"
491	"pop.n {r0-r7,pc}"
492	);
493	}
494
495	#endif
496
497	static int __kprobes
498	benchmark_pre_handler(struct kprobe *p, struct pt_regs *regs)
499	{
500	return 0;
501	}
502
503	static int benchmark(void(*fn)(void))
504	{
505	unsigned n, i, t, t0;
506
507	for (n = 1000; ; n *= 2) {
508	t0 = sched_clock();
509	for (i = n; i > 0; --i)
510	fn();
511	t = sched_clock() - t0;
512	if (t >= 250000000)
513	break; /* Stop once we took more than 0.25 seconds */
514	}
515	return t / n; /* Time for one iteration in nanoseconds */
516	};
517
518	static int kprobe_benchmark(void(*fn)(void), unsigned offset)
519	{
520	struct kprobe k = {
521	.addr = (kprobe_opcode_t *)((uintptr_t)fn + offset),
522	.pre_handler = benchmark_pre_handler,
523	};
524
525	int ret = register_kprobe(p: &k);
526	if (ret < 0) {
527	pr_err("FAIL: register_kprobe failed with %d\n", ret);
528	return ret;
529	}
530
531	ret = benchmark(fn);
532
533	unregister_kprobe(p: &k);
534	return ret;
535	};
536
537	struct benchmarks {
538	void (*fn)(void);
539	unsigned offset;
540	const char *title;
541	};
542
543	static int run_benchmarks(void)
544	{
545	int ret;
546	struct benchmarks list[] = {
547	{&benchmark_nop, 0, "nop"},
548	/*
549	* benchmark_pushpop{1,3} will have the optimised
550	* instruction emulation, whilst benchmark_pushpop{2,4} will
551	* be the equivalent unoptimised instructions.
552	*/
553	{&benchmark_pushpop1, 0, "stmdb sp!, {r3-r11,lr}"},
554	{&benchmark_pushpop1, 4, "ldmia sp!, {r3-r11,pc}"},
555	{&benchmark_pushpop2, 0, "stmdb sp!, {r0-r8,lr}"},
556	{&benchmark_pushpop2, 4, "ldmia sp!, {r0-r8,pc}"},
557	{&benchmark_pushpop3, 0, "stmdb sp!, {r4,lr}"},
558	{&benchmark_pushpop3, 4, "ldmia sp!, {r4,pc}"},
559	{&benchmark_pushpop4, 0, "stmdb sp!, {r0,lr}"},
560	{&benchmark_pushpop4, 4, "ldmia sp!, {r0,pc}"},
561	#ifdef CONFIG_THUMB2_KERNEL
562	{&benchmark_pushpop_thumb, 0, "push.n {r0-r7,lr}"},
563	{&benchmark_pushpop_thumb, 2, "pop.n {r0-r7,pc}"},
564	#endif
565	{0}
566	};
567
568	struct benchmarks *b;
569	for (b = list; b->fn; ++b) {
570	ret = kprobe_benchmark(fn: b->fn, offset: b->offset);
571	if (ret < 0)
572	return ret;
573	pr_info(" %dns for kprobe %s\n", ret, b->title);
574	}
575
576	pr_info("\n");
577	return 0;
578	}
579
580	#endif /* BENCHMARKING */
581
582
583	/*
584	* Decoding table self-consistency tests
585	*/
586
587	static const int decode_struct_sizes[NUM_DECODE_TYPES] = {
588	[DECODE_TYPE_TABLE] = sizeof(struct decode_table),
589	[DECODE_TYPE_CUSTOM] = sizeof(struct decode_custom),
590	[DECODE_TYPE_SIMULATE] = sizeof(struct decode_simulate),
591	[DECODE_TYPE_EMULATE] = sizeof(struct decode_emulate),
592	[DECODE_TYPE_OR] = sizeof(struct decode_or),
593	[DECODE_TYPE_REJECT] = sizeof(struct decode_reject)
594	};
595
596	static int table_iter(const union decode_item *table,
597	int (*fn)(const struct decode_header *, void *),
598	void *args)
599	{
600	const struct decode_header *h = (struct decode_header *)table;
601	int result;
602
603	for (;;) {
604	enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
605
606	if (type == DECODE_TYPE_END)
607	return 0;
608
609	result = fn(h, args);
610	if (result)
611	return result;
612
613	h = (struct decode_header *)
614	((uintptr_t)h + decode_struct_sizes[type]);
615
616	}
617	}
618
619	static int table_test_fail(const struct decode_header *h, const char* message)
620	{
621
622	pr_err("FAIL: kprobes test failure \"%s\" (mask %08x, value %08x)\n",
623	message, h->mask.bits, h->value.bits);
624	return -EINVAL;
625	}
626
627	struct table_test_args {
628	const union decode_item *root_table;
629	u32 parent_mask;
630	u32 parent_value;
631	};
632
633	static int table_test_fn(const struct decode_header *h, void *args)
634	{
635	struct table_test_args *a = (struct table_test_args *)args;
636	enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
637
638	if (h->value.bits & ~h->mask.bits)
639	return table_test_fail(h, message: "Match value has bits not in mask");
640
641	if ((h->mask.bits & a->parent_mask) != a->parent_mask)
642	return table_test_fail(h, message: "Mask has bits not in parent mask");
643
644	if ((h->value.bits ^ a->parent_value) & a->parent_mask)
645	return table_test_fail(h, message: "Value is inconsistent with parent");
646
647	if (type == DECODE_TYPE_TABLE) {
648	struct decode_table *d = (struct decode_table *)h;
649	struct table_test_args args2 = *a;
650	args2.parent_mask = h->mask.bits;
651	args2.parent_value = h->value.bits;
652	return table_iter(table: d->table.table, fn: table_test_fn, args: &args2);
653	}
654
655	return 0;
656	}
657
658	static int table_test(const union decode_item *table)
659	{
660	struct table_test_args args = {
661	.root_table = table,
662	.parent_mask = 0,
663	.parent_value = 0
664	};
665	return table_iter(table: args.root_table, fn: table_test_fn, args: &args);
666	}
667
668
669	/*
670	* Decoding table test coverage analysis
671	*
672	* coverage_start() builds a coverage_table which contains a list of
673	* coverage_entry's to match each entry in the specified kprobes instruction
674	* decoding table.
675	*
676	* When test cases are run, coverage_add() is called to process each case.
677	* This looks up the corresponding entry in the coverage_table and sets it as
678	* being matched, as well as clearing the regs flag appropriate for the test.
679	*
680	* After all test cases have been run, coverage_end() is called to check that
681	* all entries in coverage_table have been matched and that all regs flags are
682	* cleared. I.e. that all possible combinations of instructions described by
683	* the kprobes decoding tables have had a test case executed for them.
684	*/
685
686	bool coverage_fail;
687
688	#define MAX_COVERAGE_ENTRIES 256
689
690	struct coverage_entry {
691	const struct decode_header *header;
692	unsigned regs;
693	unsigned nesting;
694	char matched;
695	};
696
697	struct coverage_table {
698	struct coverage_entry *base;
699	unsigned num_entries;
700	unsigned nesting;
701	};
702
703	struct coverage_table coverage;
704
705	#define COVERAGE_ANY_REG (1<<0)
706	#define COVERAGE_SP (1<<1)
707	#define COVERAGE_PC (1<<2)
708	#define COVERAGE_PCWB (1<<3)
709
710	static const char coverage_register_lookup[16] = {
711	[REG_TYPE_ANY] = COVERAGE_ANY_REG | COVERAGE_SP | COVERAGE_PC,
712	[REG_TYPE_SAMEAS16] = COVERAGE_ANY_REG,
713	[REG_TYPE_SP] = COVERAGE_SP,
714	[REG_TYPE_PC] = COVERAGE_PC,
715	[REG_TYPE_NOSP] = COVERAGE_ANY_REG | COVERAGE_SP,
716	[REG_TYPE_NOSPPC] = COVERAGE_ANY_REG | COVERAGE_SP | COVERAGE_PC,
717	[REG_TYPE_NOPC] = COVERAGE_ANY_REG | COVERAGE_PC,
718	[REG_TYPE_NOPCWB] = COVERAGE_ANY_REG | COVERAGE_PC | COVERAGE_PCWB,
719	[REG_TYPE_NOPCX] = COVERAGE_ANY_REG,
720	[REG_TYPE_NOSPPCX] = COVERAGE_ANY_REG | COVERAGE_SP,
721	};
722
723	static unsigned coverage_start_registers(const struct decode_header *h)
724	{
725	unsigned regs = 0;
726	int i;
727	for (i = 0; i < 20; i += 4) {
728	int r = (h->type_regs.bits >> (DECODE_TYPE_BITS + i)) & 0xf;
729	regs |= coverage_register_lookup[r] << i;
730	}
731	return regs;
732	}
733
734	static int coverage_start_fn(const struct decode_header *h, void *args)
735	{
736	struct coverage_table *coverage = (struct coverage_table *)args;
737	enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
738	struct coverage_entry *entry = coverage->base + coverage->num_entries;
739
740	if (coverage->num_entries == MAX_COVERAGE_ENTRIES - 1) {
741	pr_err("FAIL: Out of space for test coverage data");
742	return -ENOMEM;
743	}
744
745	++coverage->num_entries;
746
747	entry->header = h;
748	entry->regs = coverage_start_registers(h);
749	entry->nesting = coverage->nesting;
750	entry->matched = false;
751
752	if (type == DECODE_TYPE_TABLE) {
753	struct decode_table *d = (struct decode_table *)h;
754	int ret;
755	++coverage->nesting;
756	ret = table_iter(table: d->table.table, fn: coverage_start_fn, args: coverage);
757	--coverage->nesting;
758	return ret;
759	}
760
761	return 0;
762	}
763
764	static int coverage_start(const union decode_item *table)
765	{
766	coverage.base = kmalloc_array(MAX_COVERAGE_ENTRIES,
767	size: sizeof(struct coverage_entry),
768	GFP_KERNEL);
769	coverage.num_entries = 0;
770	coverage.nesting = 0;
771	return table_iter(table, fn: coverage_start_fn, args: &coverage);
772	}
773
774	static void
775	coverage_add_registers(struct coverage_entry *entry, kprobe_opcode_t insn)
776	{
777	int regs = entry->header->type_regs.bits >> DECODE_TYPE_BITS;
778	int i;
779	for (i = 0; i < 20; i += 4) {
780	enum decode_reg_type reg_type = (regs >> i) & 0xf;
781	int reg = (insn >> i) & 0xf;
782	int flag;
783
784	if (!reg_type)
785	continue;
786
787	if (reg == 13)
788	flag = COVERAGE_SP;
789	else if (reg == 15)
790	flag = COVERAGE_PC;
791	else
792	flag = COVERAGE_ANY_REG;
793	entry->regs &= ~(flag << i);
794
795	switch (reg_type) {
796
797	case REG_TYPE_NONE:
798	case REG_TYPE_ANY:
799	case REG_TYPE_SAMEAS16:
800	break;
801
802	case REG_TYPE_SP:
803	if (reg != 13)
804	return;
805	break;
806
807	case REG_TYPE_PC:
808	if (reg != 15)
809	return;
810	break;
811
812	case REG_TYPE_NOSP:
813	if (reg == 13)
814	return;
815	break;
816
817	case REG_TYPE_NOSPPC:
818	case REG_TYPE_NOSPPCX:
819	if (reg == 13 || reg == 15)
820	return;
821	break;
822
823	case REG_TYPE_NOPCWB:
824	if (!is_writeback(insn))
825	break;
826	if (reg == 15) {
827	entry->regs &= ~(COVERAGE_PCWB << i);
828	return;
829	}
830	break;
831
832	case REG_TYPE_NOPC:
833	case REG_TYPE_NOPCX:
834	if (reg == 15)
835	return;
836	break;
837	}
838
839	}
840	}
841
842	static void coverage_add(kprobe_opcode_t insn)
843	{
844	struct coverage_entry *entry = coverage.base;
845	struct coverage_entry *end = coverage.base + coverage.num_entries;
846	bool matched = false;
847	unsigned nesting = 0;
848
849	for (; entry < end; ++entry) {
850	const struct decode_header *h = entry->header;
851	enum decode_type type = h->type_regs.bits & DECODE_TYPE_MASK;
852
853	if (entry->nesting > nesting)
854	continue; /* Skip sub-table we didn't match */
855
856	if (entry->nesting < nesting)
857	break; /* End of sub-table we were scanning */
858
859	if (!matched) {
860	if ((insn & h->mask.bits) != h->value.bits)
861	continue;
862	entry->matched = true;
863	}
864
865	switch (type) {
866
867	case DECODE_TYPE_TABLE:
868	++nesting;
869	break;
870
871	case DECODE_TYPE_CUSTOM:
872	case DECODE_TYPE_SIMULATE:
873	case DECODE_TYPE_EMULATE:
874	coverage_add_registers(entry, insn);
875	return;
876
877	case DECODE_TYPE_OR:
878	matched = true;
879	break;
880
881	case DECODE_TYPE_REJECT:
882	default:
883	return;
884	}
885
886	}
887	}
888
889	static void coverage_end(void)
890	{
891	struct coverage_entry *entry = coverage.base;
892	struct coverage_entry *end = coverage.base + coverage.num_entries;
893
894	for (; entry < end; ++entry) {
895	u32 mask = entry->header->mask.bits;
896	u32 value = entry->header->value.bits;
897
898	if (entry->regs) {
899	pr_err("FAIL: Register test coverage missing for %08x %08x (%05x)\n",
900	mask, value, entry->regs);
901	coverage_fail = true;
902	}
903	if (!entry->matched) {
904	pr_err("FAIL: Test coverage entry missing for %08x %08x\n",
905	mask, value);
906	coverage_fail = true;
907	}
908	}
909
910	kfree(objp: coverage.base);
911	}
912
913
914	/*
915	* Framework for instruction set test cases
916	*/
917
918	void __naked __kprobes_test_case_start(void)
919	{
920	__asm__ __volatile__ (
921	"mov r2, sp \n\t"
922	"bic r3, r2, #7 \n\t"
923	"mov sp, r3 \n\t"
924	"stmdb sp!, {r2-r11} \n\t"
925	"sub sp, sp, #"__stringify(TEST_MEMORY_SIZE)"\n\t"
926	"bic r0, lr, #1 @ r0 = inline data \n\t"
927	"mov r1, sp \n\t"
928	"bl kprobes_test_case_start \n\t"
929	RET(r0)" \n\t"
930	);
931	}
932
933	#ifndef CONFIG_THUMB2_KERNEL
934
935	void __naked __kprobes_test_case_end_32(void)
936	{
937	__asm__ __volatile__ (
938	"mov r4, lr \n\t"
939	"bl kprobes_test_case_end \n\t"
940	"cmp r0, #0 \n\t"
941	"movne pc, r0 \n\t"
942	"mov r0, r4 \n\t"
943	"add sp, sp, #"__stringify(TEST_MEMORY_SIZE)"\n\t"
944	"ldmia sp!, {r2-r11} \n\t"
945	"mov sp, r2 \n\t"
946	"mov pc, r0 \n\t"
947	);
948	}
949
950	#else /* CONFIG_THUMB2_KERNEL */
951
952	void __naked __kprobes_test_case_end_16(void)
953	{
954	__asm__ __volatile__ (
955	"mov r4, lr \n\t"
956	"bl kprobes_test_case_end \n\t"
957	"cmp r0, #0 \n\t"
958	"bxne r0 \n\t"
959	"mov r0, r4 \n\t"
960	"add sp, sp, #"__stringify(TEST_MEMORY_SIZE)"\n\t"
961	"ldmia sp!, {r2-r11} \n\t"
962	"mov sp, r2 \n\t"
963	"bx r0 \n\t"
964	);
965	}
966
967	void __naked __kprobes_test_case_end_32(void)
968	{
969	__asm__ __volatile__ (
970	".arm \n\t"
971	"orr lr, lr, #1 @ will return to Thumb code \n\t"
972	"ldr pc, 1f \n\t"
973	"1: \n\t"
974	".word __kprobes_test_case_end_16 \n\t"
975	);
976	}
977
978	#endif
979
980
981	int kprobe_test_flags;
982	int kprobe_test_cc_position;
983
984	static int test_try_count;
985	static int test_pass_count;
986	static int test_fail_count;
987
988	static struct pt_regs initial_regs;
989	static struct pt_regs expected_regs;
990	static struct pt_regs result_regs;
991
992	static u32 expected_memory[TEST_MEMORY_SIZE/sizeof(u32)];
993
994	static const char *current_title;
995	static struct test_arg *current_args;
996	static u32 *current_stack;
997	static uintptr_t current_branch_target;
998
999	static uintptr_t current_code_start;
1000	static kprobe_opcode_t current_instruction;
1001
1002
1003	#define TEST_CASE_PASSED -1
1004	#define TEST_CASE_FAILED -2
1005
1006	static int test_case_run_count;
1007	static bool test_case_is_thumb;
1008	static int test_instance;
1009
1010	static unsigned long test_check_cc(int cc, unsigned long cpsr)
1011	{
1012	int ret = arm_check_condition(cc << 28, cpsr);
1013
1014	return (ret != ARM_OPCODE_CONDTEST_FAIL);
1015	}
1016
1017	static int is_last_scenario;
1018	static int probe_should_run; /* 0 = no, 1 = yes, -1 = unknown */
1019	static int memory_needs_checking;
1020
1021	static unsigned long test_context_cpsr(int scenario)
1022	{
1023	unsigned long cpsr;
1024
1025	probe_should_run = 1;
1026
1027	/* Default case is that we cycle through 16 combinations of flags */
1028	cpsr = (scenario & 0xf) << 28; /* N,Z,C,V flags */
1029	cpsr |= (scenario & 0xf) << 16; /* GE flags */
1030	cpsr |= (scenario & 0x1) << 27; /* Toggle Q flag */
1031
1032	if (!test_case_is_thumb) {
1033	/* Testing ARM code */
1034	int cc = current_instruction >> 28;
1035
1036	probe_should_run = test_check_cc(cc, cpsr) != 0;
1037	if (scenario == 15)
1038	is_last_scenario = true;
1039
1040	} else if (kprobe_test_flags & TEST_FLAG_NO_ITBLOCK) {
1041	/* Testing Thumb code without setting ITSTATE */
1042	if (kprobe_test_cc_position) {
1043	int cc = (current_instruction >> kprobe_test_cc_position) & 0xf;
1044	probe_should_run = test_check_cc(cc, cpsr) != 0;
1045	}
1046
1047	if (scenario == 15)
1048	is_last_scenario = true;
1049
1050	} else if (kprobe_test_flags & TEST_FLAG_FULL_ITBLOCK) {
1051	/* Testing Thumb code with all combinations of ITSTATE */
1052	unsigned x = (scenario >> 4);
1053	unsigned cond_base = x % 7; /* ITSTATE<7:5> */
1054	unsigned mask = x / 7 + 2; /* ITSTATE<4:0>, bits reversed */
1055
1056	if (mask > 0x1f) {
1057	/* Finish by testing state from instruction 'itt al' */
1058	cond_base = 7;
1059	mask = 0x4;
1060	if ((scenario & 0xf) == 0xf)
1061	is_last_scenario = true;
1062	}
1063
1064	cpsr |= cond_base << 13; /* ITSTATE<7:5> */
1065	cpsr |= (mask & 0x1) << 12; /* ITSTATE<4> */
1066	cpsr |= (mask & 0x2) << 10; /* ITSTATE<3> */
1067	cpsr |= (mask & 0x4) << 8; /* ITSTATE<2> */
1068	cpsr |= (mask & 0x8) << 23; /* ITSTATE<1> */
1069	cpsr |= (mask & 0x10) << 21; /* ITSTATE<0> */
1070
1071	probe_should_run = test_check_cc(cc: (cpsr >> 12) & 0xf, cpsr) != 0;
1072
1073	} else {
1074	/* Testing Thumb code with several combinations of ITSTATE */
1075	switch (scenario) {
1076	case 16: /* Clear NZCV flags and 'it eq' state (false as Z=0) */
1077	cpsr = 0x00000800;
1078	probe_should_run = 0;
1079	break;
1080	case 17: /* Set NZCV flags and 'it vc' state (false as V=1) */
1081	cpsr = 0xf0007800;
1082	probe_should_run = 0;
1083	break;
1084	case 18: /* Clear NZCV flags and 'it ls' state (true as C=0) */
1085	cpsr = 0x00009800;
1086	break;
1087	case 19: /* Set NZCV flags and 'it cs' state (true as C=1) */
1088	cpsr = 0xf0002800;
1089	is_last_scenario = true;
1090	break;
1091	}
1092	}
1093
1094	return cpsr;
1095	}
1096
1097	static void setup_test_context(struct pt_regs *regs)
1098	{
1099	int scenario = test_case_run_count>>1;
1100	unsigned long val;
1101	struct test_arg *args;
1102	int i;
1103
1104	is_last_scenario = false;
1105	memory_needs_checking = false;
1106
1107	/* Initialise test memory on stack */
1108	val = (scenario & 1) ? VALM : ~VALM;
1109	for (i = 0; i < TEST_MEMORY_SIZE / sizeof(current_stack[0]); ++i)
1110	current_stack[i] = val + (i << 8);
1111	/* Put target of branch on stack for tests which load PC from memory */
1112	if (current_branch_target)
1113	current_stack[15] = current_branch_target;
1114	/* Put a value for SP on stack for tests which load SP from memory */
1115	current_stack[13] = (u32)current_stack + 120;
1116
1117	/* Initialise register values to their default state */
1118	val = (scenario & 2) ? VALR : ~VALR;
1119	for (i = 0; i < 13; ++i)
1120	regs->uregs[i] = val ^ (i << 8);
1121	regs->ARM_lr = val ^ (14 << 8);
1122	regs->ARM_cpsr &= ~(APSR_MASK | PSR_IT_MASK);
1123	regs->ARM_cpsr |= test_context_cpsr(scenario);
1124
1125	/* Perform testcase specific register setup */
1126	args = current_args;
1127	for (; args[0].type != ARG_TYPE_END; ++args)
1128	switch (args[0].type) {
1129	case ARG_TYPE_REG: {
1130	struct test_arg_regptr *arg =
1131	(struct test_arg_regptr *)args;
1132	regs->uregs[arg->reg] = arg->val;
1133	break;
1134	}
1135	case ARG_TYPE_PTR: {
1136	struct test_arg_regptr *arg =
1137	(struct test_arg_regptr *)args;
1138	regs->uregs[arg->reg] =
1139	(unsigned long)current_stack + arg->val;
1140	memory_needs_checking = true;
1141	/*
1142	* Test memory at an address below SP is in danger of
1143	* being altered by an interrupt occurring and pushing
1144	* data onto the stack. Disable interrupts to stop this.
1145	*/
1146	if (arg->reg == 13)
1147	regs->ARM_cpsr |= PSR_I_BIT;
1148	break;
1149	}
1150	case ARG_TYPE_MEM: {
1151	struct test_arg_mem *arg = (struct test_arg_mem *)args;
1152	current_stack[arg->index] = arg->val;
1153	break;
1154	}
1155	default:
1156	break;
1157	}
1158	}
1159
1160	struct test_probe {
1161	struct kprobe kprobe;
1162	bool registered;
1163	int hit;
1164	};
1165
1166	static void unregister_test_probe(struct test_probe *probe)
1167	{
1168	if (probe->registered) {
1169	unregister_kprobe(p: &probe->kprobe);
1170	probe->kprobe.flags = 0; /* Clear disable flag to allow reuse */
1171	}
1172	probe->registered = false;
1173	}
1174
1175	static int register_test_probe(struct test_probe *probe)
1176	{
1177	int ret;
1178
1179	if (probe->registered)
1180	BUG();
1181
1182	ret = register_kprobe(p: &probe->kprobe);
1183	if (ret >= 0) {
1184	probe->registered = true;
1185	probe->hit = -1;
1186	}
1187	return ret;
1188	}
1189
1190	static int __kprobes
1191	test_before_pre_handler(struct kprobe *p, struct pt_regs *regs)
1192	{
1193	container_of(p, struct test_probe, kprobe)->hit = test_instance;
1194	return 0;
1195	}
1196
1197	static void __kprobes
1198	test_before_post_handler(struct kprobe *p, struct pt_regs *regs,
1199	unsigned long flags)
1200	{
1201	setup_test_context(regs);
1202	initial_regs = *regs;
1203	initial_regs.ARM_cpsr &= ~PSR_IGNORE_BITS;
1204	}
1205
1206	static int __kprobes
1207	test_case_pre_handler(struct kprobe *p, struct pt_regs *regs)
1208	{
1209	container_of(p, struct test_probe, kprobe)->hit = test_instance;
1210	return 0;
1211	}
1212
1213	static int __kprobes
1214	test_after_pre_handler(struct kprobe *p, struct pt_regs *regs)
1215	{
1216	struct test_arg *args;
1217
1218	if (container_of(p, struct test_probe, kprobe)->hit == test_instance)
1219	return 0; /* Already run for this test instance */
1220
1221	result_regs = *regs;
1222
1223	/* Mask out results which are indeterminate */
1224	result_regs.ARM_cpsr &= ~PSR_IGNORE_BITS;
1225	for (args = current_args; args[0].type != ARG_TYPE_END; ++args)
1226	if (args[0].type == ARG_TYPE_REG_MASKED) {
1227	struct test_arg_regptr *arg =
1228	(struct test_arg_regptr *)args;
1229	result_regs.uregs[arg->reg] &= arg->val;
1230	}
1231
1232	/* Undo any changes done to SP by the test case */
1233	regs->ARM_sp = (unsigned long)current_stack;
1234	/* Enable interrupts in case setup_test_context disabled them */
1235	regs->ARM_cpsr &= ~PSR_I_BIT;
1236
1237	container_of(p, struct test_probe, kprobe)->hit = test_instance;
1238	return 0;
1239	}
1240
1241	static struct test_probe test_before_probe = {
1242	.kprobe.pre_handler = test_before_pre_handler,
1243	.kprobe.post_handler = test_before_post_handler,
1244	};
1245
1246	static struct test_probe test_case_probe = {
1247	.kprobe.pre_handler = test_case_pre_handler,
1248	};
1249
1250	static struct test_probe test_after_probe = {
1251	.kprobe.pre_handler = test_after_pre_handler,
1252	};
1253
1254	static struct test_probe test_after2_probe = {
1255	.kprobe.pre_handler = test_after_pre_handler,
1256	};
1257
1258	static void test_case_cleanup(void)
1259	{
1260	unregister_test_probe(probe: &test_before_probe);
1261	unregister_test_probe(probe: &test_case_probe);
1262	unregister_test_probe(probe: &test_after_probe);
1263	unregister_test_probe(probe: &test_after2_probe);
1264	}
1265
1266	static void print_registers(struct pt_regs *regs)
1267	{
1268	pr_err("r0 %08lx | r1 %08lx | r2 %08lx | r3 %08lx\n",
1269	regs->ARM_r0, regs->ARM_r1, regs->ARM_r2, regs->ARM_r3);
1270	pr_err("r4 %08lx | r5 %08lx | r6 %08lx | r7 %08lx\n",
1271	regs->ARM_r4, regs->ARM_r5, regs->ARM_r6, regs->ARM_r7);
1272	pr_err("r8 %08lx | r9 %08lx | r10 %08lx | r11 %08lx\n",
1273	regs->ARM_r8, regs->ARM_r9, regs->ARM_r10, regs->ARM_fp);
1274	pr_err("r12 %08lx | sp %08lx | lr %08lx | pc %08lx\n",
1275	regs->ARM_ip, regs->ARM_sp, regs->ARM_lr, regs->ARM_pc);
1276	pr_err("cpsr %08lx\n", regs->ARM_cpsr);
1277	}
1278
1279	static void print_memory(u32 *mem, size_t size)
1280	{
1281	int i;
1282	for (i = 0; i < size / sizeof(u32); i += 4)
1283	pr_err("%08x %08x %08x %08x\n", mem[i], mem[i+1],
1284	mem[i+2], mem[i+3]);
1285	}
1286
1287	static size_t expected_memory_size(u32 *sp)
1288	{
1289	size_t size = sizeof(expected_memory);
1290	int offset = (uintptr_t)sp - (uintptr_t)current_stack;
1291	if (offset > 0)
1292	size -= offset;
1293	return size;
1294	}
1295
1296	static void test_case_failed(const char *message)
1297	{
1298	test_case_cleanup();
1299
1300	pr_err("FAIL: %s\n", message);
1301	pr_err("FAIL: Test %s\n", current_title);
1302	pr_err("FAIL: Scenario %d\n", test_case_run_count >> 1);
1303	}
1304
1305	static unsigned long next_instruction(unsigned long pc)
1306	{
1307	#ifdef CONFIG_THUMB2_KERNEL
1308	if ((pc & 1) &&
1309	!is_wide_instruction(__mem_to_opcode_thumb16(*(u16 *)(pc - 1))))
1310	return pc + 2;
1311	else
1312	#endif
1313	return pc + 4;
1314	}
1315
1316	static uintptr_t __used kprobes_test_case_start(const char **title, void *stack)
1317	{
1318	struct test_arg *args;
1319	struct test_arg_end *end_arg;
1320	unsigned long test_code;
1321
1322	current_title = *title++;
1323	args = (struct test_arg *)title;
1324	current_args = args;
1325	current_stack = stack;
1326
1327	++test_try_count;
1328
1329	while (args->type != ARG_TYPE_END)
1330	++args;
1331	end_arg = (struct test_arg_end *)args;
1332
1333	test_code = (unsigned long)(args + 1); /* Code starts after args */
1334
1335	test_case_is_thumb = end_arg->flags & ARG_FLAG_THUMB;
1336	if (test_case_is_thumb)
1337	test_code |= 1;
1338
1339	current_code_start = test_code;
1340
1341	current_branch_target = 0;
1342	if (end_arg->branch_offset != end_arg->end_offset)
1343	current_branch_target = test_code + end_arg->branch_offset;
1344
1345	test_code += end_arg->code_offset;
1346	test_before_probe.kprobe.addr = (kprobe_opcode_t *)test_code;
1347
1348	test_code = next_instruction(pc: test_code);
1349	test_case_probe.kprobe.addr = (kprobe_opcode_t *)test_code;
1350
1351	if (test_case_is_thumb) {
1352	u16 *p = (u16 *)(test_code & ~1);
1353	current_instruction = __mem_to_opcode_thumb16(p[0]);
1354	if (is_wide_instruction(current_instruction)) {
1355	u16 instr2 = __mem_to_opcode_thumb16(p[1]);
1356	current_instruction = __opcode_thumb32_compose(current_instruction, instr2);
1357	}
1358	} else {
1359	current_instruction = __mem_to_opcode_arm(*(u32 *)test_code);
1360	}
1361
1362	if (current_title[0] == '.')
1363	verbose("%s\n", current_title);
1364	else
1365	verbose("%s\t@ %0*x\n", current_title,
1366	test_case_is_thumb ? 4 : 8,
1367	current_instruction);
1368
1369	test_code = next_instruction(pc: test_code);
1370	test_after_probe.kprobe.addr = (kprobe_opcode_t *)test_code;
1371
1372	if (kprobe_test_flags & TEST_FLAG_NARROW_INSTR) {
1373	if (!test_case_is_thumb ||
1374	is_wide_instruction(current_instruction)) {
1375	test_case_failed(message: "expected 16-bit instruction");
1376	goto fail;
1377	}
1378	} else {
1379	if (test_case_is_thumb &&
1380	!is_wide_instruction(current_instruction)) {
1381	test_case_failed(message: "expected 32-bit instruction");
1382	goto fail;
1383	}
1384	}
1385
1386	coverage_add(insn: current_instruction);
1387
1388	if (end_arg->flags & ARG_FLAG_UNSUPPORTED) {
1389	if (register_test_probe(probe: &test_case_probe) < 0)
1390	goto pass;
1391	test_case_failed(message: "registered probe for unsupported instruction");
1392	goto fail;
1393	}
1394
1395	if (end_arg->flags & ARG_FLAG_SUPPORTED) {
1396	if (register_test_probe(probe: &test_case_probe) >= 0)
1397	goto pass;
1398	test_case_failed(message: "couldn't register probe for supported instruction");
1399	goto fail;
1400	}
1401
1402	if (register_test_probe(probe: &test_before_probe) < 0) {
1403	test_case_failed(message: "register test_before_probe failed");
1404	goto fail;
1405	}
1406	if (register_test_probe(probe: &test_after_probe) < 0) {
1407	test_case_failed(message: "register test_after_probe failed");
1408	goto fail;
1409	}
1410	if (current_branch_target) {
1411	test_after2_probe.kprobe.addr =
1412	(kprobe_opcode_t *)current_branch_target;
1413	if (register_test_probe(probe: &test_after2_probe) < 0) {
1414	test_case_failed(message: "register test_after2_probe failed");
1415	goto fail;
1416	}
1417	}
1418
1419	/* Start first run of test case */
1420	test_case_run_count = 0;
1421	++test_instance;
1422	return current_code_start;
1423	pass:
1424	test_case_run_count = TEST_CASE_PASSED;
1425	return (uintptr_t)test_after_probe.kprobe.addr;
1426	fail:
1427	test_case_run_count = TEST_CASE_FAILED;
1428	return (uintptr_t)test_after_probe.kprobe.addr;
1429	}
1430
1431	static bool check_test_results(void)
1432	{
1433	size_t mem_size = 0;
1434	u32 *mem = 0;
1435
1436	if (memcmp(p: &expected_regs, q: &result_regs, size: sizeof(expected_regs))) {
1437	test_case_failed(message: "registers differ");
1438	goto fail;
1439	}
1440
1441	if (memory_needs_checking) {
1442	mem = (u32 *)result_regs.ARM_sp;
1443	mem_size = expected_memory_size(sp: mem);
1444	if (memcmp(p: expected_memory, q: mem, size: mem_size)) {
1445	test_case_failed(message: "test memory differs");
1446	goto fail;
1447	}
1448	}
1449
1450	return true;
1451
1452	fail:
1453	pr_err("initial_regs:\n");
1454	print_registers(regs: &initial_regs);
1455	pr_err("expected_regs:\n");
1456	print_registers(regs: &expected_regs);
1457	pr_err("result_regs:\n");
1458	print_registers(regs: &result_regs);
1459
1460	if (mem) {
1461	pr_err("expected_memory:\n");
1462	print_memory(mem: expected_memory, size: mem_size);
1463	pr_err("result_memory:\n");
1464	print_memory(mem, size: mem_size);
1465	}
1466
1467	return false;
1468	}
1469
1470	static uintptr_t __used kprobes_test_case_end(void)
1471	{
1472	if (test_case_run_count < 0) {
1473	if (test_case_run_count == TEST_CASE_PASSED)
1474	/* kprobes_test_case_start did all the needed testing */
1475	goto pass;
1476	else
1477	/* kprobes_test_case_start failed */
1478	goto fail;
1479	}
1480
1481	if (test_before_probe.hit != test_instance) {
1482	test_case_failed(message: "test_before_handler not run");
1483	goto fail;
1484	}
1485
1486	if (test_after_probe.hit != test_instance &&
1487	test_after2_probe.hit != test_instance) {
1488	test_case_failed(message: "test_after_handler not run");
1489	goto fail;
1490	}
1491
1492	/*
1493	* Even numbered test runs ran without a probe on the test case so
1494	* we can gather reference results. The subsequent odd numbered run
1495	* will have the probe inserted.
1496	*/
1497	if ((test_case_run_count & 1) == 0) {
1498	/* Save results from run without probe */
1499	u32 *mem = (u32 *)result_regs.ARM_sp;
1500	expected_regs = result_regs;
1501	memcpy(expected_memory, mem, expected_memory_size(mem));
1502
1503	/* Insert probe onto test case instruction */
1504	if (register_test_probe(probe: &test_case_probe) < 0) {
1505	test_case_failed(message: "register test_case_probe failed");
1506	goto fail;
1507	}
1508	} else {
1509	/* Check probe ran as expected */
1510	if (probe_should_run == 1) {
1511	if (test_case_probe.hit != test_instance) {
1512	test_case_failed(message: "test_case_handler not run");
1513	goto fail;
1514	}
1515	} else if (probe_should_run == 0) {
1516	if (test_case_probe.hit == test_instance) {
1517	test_case_failed(message: "test_case_handler ran");
1518	goto fail;
1519	}
1520	}
1521
1522	/* Remove probe for any subsequent reference run */
1523	unregister_test_probe(probe: &test_case_probe);
1524
1525	if (!check_test_results())
1526	goto fail;
1527
1528	if (is_last_scenario)
1529	goto pass;
1530	}
1531
1532	/* Do next test run */
1533	++test_case_run_count;
1534	++test_instance;
1535	return current_code_start;
1536	fail:
1537	++test_fail_count;
1538	goto end;
1539	pass:
1540	++test_pass_count;
1541	end:
1542	test_case_cleanup();
1543	return 0;
1544	}
1545
1546
1547	/*
1548	* Top level test functions
1549	*/
1550
1551	static int run_test_cases(void (*tests)(void), const union decode_item *table)
1552	{
1553	int ret;
1554
1555	pr_info(" Check decoding tables\n");
1556	ret = table_test(table);
1557	if (ret)
1558	return ret;
1559
1560	pr_info(" Run test cases\n");
1561	ret = coverage_start(table);
1562	if (ret)
1563	return ret;
1564
1565	tests();
1566
1567	coverage_end();
1568	return 0;
1569	}
1570
1571
1572	static int __init run_all_tests(void)
1573	{
1574	int ret = 0;
1575
1576	pr_info("Beginning kprobe tests...\n");
1577
1578	#ifndef CONFIG_THUMB2_KERNEL
1579
1580	pr_info("Probe ARM code\n");
1581	ret = run_api_tests(func: arm_func);
1582	if (ret)
1583	goto out;
1584
1585	pr_info("ARM instruction simulation\n");
1586	ret = run_test_cases(tests: kprobe_arm_test_cases, table: probes_decode_arm_table);
1587	if (ret)
1588	goto out;
1589
1590	#else /* CONFIG_THUMB2_KERNEL */
1591
1592	pr_info("Probe 16-bit Thumb code\n");
1593	ret = run_api_tests(thumb16_func);
1594	if (ret)
1595	goto out;
1596
1597	pr_info("Probe 32-bit Thumb code, even halfword\n");
1598	ret = run_api_tests(thumb32even_func);
1599	if (ret)
1600	goto out;
1601
1602	pr_info("Probe 32-bit Thumb code, odd halfword\n");
1603	ret = run_api_tests(thumb32odd_func);
1604	if (ret)
1605	goto out;
1606
1607	pr_info("16-bit Thumb instruction simulation\n");
1608	ret = run_test_cases(kprobe_thumb16_test_cases,
1609	probes_decode_thumb16_table);
1610	if (ret)
1611	goto out;
1612
1613	pr_info("32-bit Thumb instruction simulation\n");
1614	ret = run_test_cases(kprobe_thumb32_test_cases,
1615	probes_decode_thumb32_table);
1616	if (ret)
1617	goto out;
1618	#endif
1619
1620	pr_info("Total instruction simulation tests=%d, pass=%d fail=%d\n",
1621	test_try_count, test_pass_count, test_fail_count);
1622	if (test_fail_count) {
1623	ret = -EINVAL;
1624	goto out;
1625	}
1626
1627	#if BENCHMARKING
1628	pr_info("Benchmarks\n");
1629	ret = run_benchmarks();
1630	if (ret)
1631	goto out;
1632	#endif
1633
1634	#if __LINUX_ARM_ARCH__ >= 7
1635	/* We are able to run all test cases so coverage should be complete */
1636	if (coverage_fail) {
1637	pr_err("FAIL: Test coverage checks failed\n");
1638	ret = -EINVAL;
1639	goto out;
1640	}
1641	#endif
1642
1643	out:
1644	if (ret == 0)
1645	ret = tests_failed;
1646	if (ret == 0)
1647	pr_info("Finished kprobe tests OK\n");
1648	else
1649	pr_err("kprobe tests failed\n");
1650
1651	return ret;
1652	}
1653
1654
1655	/*
1656	* Module setup
1657	*/
1658
1659	#ifdef MODULE
1660
1661	static void __exit kprobe_test_exit(void)
1662	{
1663	}
1664
1665	module_init(run_all_tests)
1666	module_exit(kprobe_test_exit)
1667	MODULE_LICENSE("GPL");
1668
1669	#else /* !MODULE */
1670
1671	late_initcall(run_all_tests);
1672
1673	#endif
1674