1/*
2 * Kernel Probes (KProbes)
3 * kernel/kprobes.c
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * Copyright (C) IBM Corporation, 2002, 2004
20 *
21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22 * Probes initial implementation (includes suggestions from
23 * Rusty Russell).
24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25 * hlists and exceptions notifier as suggested by Andi Kleen.
26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27 * interface to access function arguments.
28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29 * exceptions notifier to be first on the priority list.
30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32 * <prasanna@in.ibm.com> added function-return probes.
33 */
34#include <linux/kprobes.h>
35#include <linux/hash.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/stddef.h>
39#include <linux/export.h>
40#include <linux/moduleloader.h>
41#include <linux/kallsyms.h>
42#include <linux/freezer.h>
43#include <linux/seq_file.h>
44#include <linux/debugfs.h>
45#include <linux/sysctl.h>
46#include <linux/kdebug.h>
47#include <linux/memory.h>
48#include <linux/ftrace.h>
49#include <linux/cpu.h>
50#include <linux/jump_label.h>
51
52#include <asm/sections.h>
53#include <asm/cacheflush.h>
54#include <asm/errno.h>
55#include <linux/uaccess.h>
56
57#define KPROBE_HASH_BITS 6
58#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
59
60
61static int kprobes_initialized;
62static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
63static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
64
65/* NOTE: change this value only with kprobe_mutex held */
66static bool kprobes_all_disarmed;
67
68/* This protects kprobe_table and optimizing_list */
69static DEFINE_MUTEX(kprobe_mutex);
70static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
71static struct {
72 raw_spinlock_t lock ____cacheline_aligned_in_smp;
73} kretprobe_table_locks[KPROBE_TABLE_SIZE];
74
75kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
76 unsigned int __unused)
77{
78 return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
79}
80
81static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
82{
83 return &(kretprobe_table_locks[hash].lock);
84}
85
86/* Blacklist -- list of struct kprobe_blacklist_entry */
87static LIST_HEAD(kprobe_blacklist);
88
89#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
90/*
91 * kprobe->ainsn.insn points to the copy of the instruction to be
92 * single-stepped. x86_64, POWER4 and above have no-exec support and
93 * stepping on the instruction on a vmalloced/kmalloced/data page
94 * is a recipe for disaster
95 */
96struct kprobe_insn_page {
97 struct list_head list;
98 kprobe_opcode_t *insns; /* Page of instruction slots */
99 struct kprobe_insn_cache *cache;
100 int nused;
101 int ngarbage;
102 char slot_used[];
103};
104
105#define KPROBE_INSN_PAGE_SIZE(slots) \
106 (offsetof(struct kprobe_insn_page, slot_used) + \
107 (sizeof(char) * (slots)))
108
109static int slots_per_page(struct kprobe_insn_cache *c)
110{
111 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
112}
113
114enum kprobe_slot_state {
115 SLOT_CLEAN = 0,
116 SLOT_DIRTY = 1,
117 SLOT_USED = 2,
118};
119
120void __weak *alloc_insn_page(void)
121{
122 return module_alloc(PAGE_SIZE);
123}
124
125void __weak free_insn_page(void *page)
126{
127 module_memfree(page);
128}
129
130struct kprobe_insn_cache kprobe_insn_slots = {
131 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
132 .alloc = alloc_insn_page,
133 .free = free_insn_page,
134 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
135 .insn_size = MAX_INSN_SIZE,
136 .nr_garbage = 0,
137};
138static int collect_garbage_slots(struct kprobe_insn_cache *c);
139
140/**
141 * __get_insn_slot() - Find a slot on an executable page for an instruction.
142 * We allocate an executable page if there's no room on existing ones.
143 */
144kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c)
145{
146 struct kprobe_insn_page *kip;
147 kprobe_opcode_t *slot = NULL;
148
149 /* Since the slot array is not protected by rcu, we need a mutex */
150 mutex_lock(&c->mutex);
151 retry:
152 rcu_read_lock();
153 list_for_each_entry_rcu(kip, &c->pages, list) {
154 if (kip->nused < slots_per_page(c)) {
155 int i;
156 for (i = 0; i < slots_per_page(c); i++) {
157 if (kip->slot_used[i] == SLOT_CLEAN) {
158 kip->slot_used[i] = SLOT_USED;
159 kip->nused++;
160 slot = kip->insns + (i * c->insn_size);
161 rcu_read_unlock();
162 goto out;
163 }
164 }
165 /* kip->nused is broken. Fix it. */
166 kip->nused = slots_per_page(c);
167 WARN_ON(1);
168 }
169 }
170 rcu_read_unlock();
171
172 /* If there are any garbage slots, collect it and try again. */
173 if (c->nr_garbage && collect_garbage_slots(c) == 0)
174 goto retry;
175
176 /* All out of space. Need to allocate a new page. */
177 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
178 if (!kip)
179 goto out;
180
181 /*
182 * Use module_alloc so this page is within +/- 2GB of where the
183 * kernel image and loaded module images reside. This is required
184 * so x86_64 can correctly handle the %rip-relative fixups.
185 */
186 kip->insns = c->alloc();
187 if (!kip->insns) {
188 kfree(kip);
189 goto out;
190 }
191 INIT_LIST_HEAD(&kip->list);
192 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
193 kip->slot_used[0] = SLOT_USED;
194 kip->nused = 1;
195 kip->ngarbage = 0;
196 kip->cache = c;
197 list_add_rcu(&kip->list, &c->pages);
198 slot = kip->insns;
199out:
200 mutex_unlock(&c->mutex);
201 return slot;
202}
203
204/* Return 1 if all garbages are collected, otherwise 0. */
205static int collect_one_slot(struct kprobe_insn_page *kip, int idx)
206{
207 kip->slot_used[idx] = SLOT_CLEAN;
208 kip->nused--;
209 if (kip->nused == 0) {
210 /*
211 * Page is no longer in use. Free it unless
212 * it's the last one. We keep the last one
213 * so as not to have to set it up again the
214 * next time somebody inserts a probe.
215 */
216 if (!list_is_singular(&kip->list)) {
217 list_del_rcu(&kip->list);
218 synchronize_rcu();
219 kip->cache->free(kip->insns);
220 kfree(kip);
221 }
222 return 1;
223 }
224 return 0;
225}
226
227static int collect_garbage_slots(struct kprobe_insn_cache *c)
228{
229 struct kprobe_insn_page *kip, *next;
230
231 /* Ensure no-one is interrupted on the garbages */
232 synchronize_sched();
233
234 list_for_each_entry_safe(kip, next, &c->pages, list) {
235 int i;
236 if (kip->ngarbage == 0)
237 continue;
238 kip->ngarbage = 0; /* we will collect all garbages */
239 for (i = 0; i < slots_per_page(c); i++) {
240 if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
241 break;
242 }
243 }
244 c->nr_garbage = 0;
245 return 0;
246}
247
248void __free_insn_slot(struct kprobe_insn_cache *c,
249 kprobe_opcode_t *slot, int dirty)
250{
251 struct kprobe_insn_page *kip;
252 long idx;
253
254 mutex_lock(&c->mutex);
255 rcu_read_lock();
256 list_for_each_entry_rcu(kip, &c->pages, list) {
257 idx = ((long)slot - (long)kip->insns) /
258 (c->insn_size * sizeof(kprobe_opcode_t));
259 if (idx >= 0 && idx < slots_per_page(c))
260 goto out;
261 }
262 /* Could not find this slot. */
263 WARN_ON(1);
264 kip = NULL;
265out:
266 rcu_read_unlock();
267 /* Mark and sweep: this may sleep */
268 if (kip) {
269 /* Check double free */
270 WARN_ON(kip->slot_used[idx] != SLOT_USED);
271 if (dirty) {
272 kip->slot_used[idx] = SLOT_DIRTY;
273 kip->ngarbage++;
274 if (++c->nr_garbage > slots_per_page(c))
275 collect_garbage_slots(c);
276 } else {
277 collect_one_slot(kip, idx);
278 }
279 }
280 mutex_unlock(&c->mutex);
281}
282
283/*
284 * Check given address is on the page of kprobe instruction slots.
285 * This will be used for checking whether the address on a stack
286 * is on a text area or not.
287 */
288bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
289{
290 struct kprobe_insn_page *kip;
291 bool ret = false;
292
293 rcu_read_lock();
294 list_for_each_entry_rcu(kip, &c->pages, list) {
295 if (addr >= (unsigned long)kip->insns &&
296 addr < (unsigned long)kip->insns + PAGE_SIZE) {
297 ret = true;
298 break;
299 }
300 }
301 rcu_read_unlock();
302
303 return ret;
304}
305
306#ifdef CONFIG_OPTPROBES
307/* For optimized_kprobe buffer */
308struct kprobe_insn_cache kprobe_optinsn_slots = {
309 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
310 .alloc = alloc_insn_page,
311 .free = free_insn_page,
312 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
313 /* .insn_size is initialized later */
314 .nr_garbage = 0,
315};
316#endif
317#endif
318
319/* We have preemption disabled.. so it is safe to use __ versions */
320static inline void set_kprobe_instance(struct kprobe *kp)
321{
322 __this_cpu_write(kprobe_instance, kp);
323}
324
325static inline void reset_kprobe_instance(void)
326{
327 __this_cpu_write(kprobe_instance, NULL);
328}
329
330/*
331 * This routine is called either:
332 * - under the kprobe_mutex - during kprobe_[un]register()
333 * OR
334 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
335 */
336struct kprobe *get_kprobe(void *addr)
337{
338 struct hlist_head *head;
339 struct kprobe *p;
340
341 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
342 hlist_for_each_entry_rcu(p, head, hlist) {
343 if (p->addr == addr)
344 return p;
345 }
346
347 return NULL;
348}
349NOKPROBE_SYMBOL(get_kprobe);
350
351static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
352
353/* Return true if the kprobe is an aggregator */
354static inline int kprobe_aggrprobe(struct kprobe *p)
355{
356 return p->pre_handler == aggr_pre_handler;
357}
358
359/* Return true(!0) if the kprobe is unused */
360static inline int kprobe_unused(struct kprobe *p)
361{
362 return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
363 list_empty(&p->list);
364}
365
366/*
367 * Keep all fields in the kprobe consistent
368 */
369static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
370{
371 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
372 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
373}
374
375#ifdef CONFIG_OPTPROBES
376/* NOTE: change this value only with kprobe_mutex held */
377static bool kprobes_allow_optimization;
378
379/*
380 * Call all pre_handler on the list, but ignores its return value.
381 * This must be called from arch-dep optimized caller.
382 */
383void opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
384{
385 struct kprobe *kp;
386
387 list_for_each_entry_rcu(kp, &p->list, list) {
388 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
389 set_kprobe_instance(kp);
390 kp->pre_handler(kp, regs);
391 }
392 reset_kprobe_instance();
393 }
394}
395NOKPROBE_SYMBOL(opt_pre_handler);
396
397/* Free optimized instructions and optimized_kprobe */
398static void free_aggr_kprobe(struct kprobe *p)
399{
400 struct optimized_kprobe *op;
401
402 op = container_of(p, struct optimized_kprobe, kp);
403 arch_remove_optimized_kprobe(op);
404 arch_remove_kprobe(p);
405 kfree(op);
406}
407
408/* Return true(!0) if the kprobe is ready for optimization. */
409static inline int kprobe_optready(struct kprobe *p)
410{
411 struct optimized_kprobe *op;
412
413 if (kprobe_aggrprobe(p)) {
414 op = container_of(p, struct optimized_kprobe, kp);
415 return arch_prepared_optinsn(&op->optinsn);
416 }
417
418 return 0;
419}
420
421/* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */
422static inline int kprobe_disarmed(struct kprobe *p)
423{
424 struct optimized_kprobe *op;
425
426 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */
427 if (!kprobe_aggrprobe(p))
428 return kprobe_disabled(p);
429
430 op = container_of(p, struct optimized_kprobe, kp);
431
432 return kprobe_disabled(p) && list_empty(&op->list);
433}
434
435/* Return true(!0) if the probe is queued on (un)optimizing lists */
436static int kprobe_queued(struct kprobe *p)
437{
438 struct optimized_kprobe *op;
439
440 if (kprobe_aggrprobe(p)) {
441 op = container_of(p, struct optimized_kprobe, kp);
442 if (!list_empty(&op->list))
443 return 1;
444 }
445 return 0;
446}
447
448/*
449 * Return an optimized kprobe whose optimizing code replaces
450 * instructions including addr (exclude breakpoint).
451 */
452static struct kprobe *get_optimized_kprobe(unsigned long addr)
453{
454 int i;
455 struct kprobe *p = NULL;
456 struct optimized_kprobe *op;
457
458 /* Don't check i == 0, since that is a breakpoint case. */
459 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
460 p = get_kprobe((void *)(addr - i));
461
462 if (p && kprobe_optready(p)) {
463 op = container_of(p, struct optimized_kprobe, kp);
464 if (arch_within_optimized_kprobe(op, addr))
465 return p;
466 }
467
468 return NULL;
469}
470
471/* Optimization staging list, protected by kprobe_mutex */
472static LIST_HEAD(optimizing_list);
473static LIST_HEAD(unoptimizing_list);
474static LIST_HEAD(freeing_list);
475
476static void kprobe_optimizer(struct work_struct *work);
477static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
478#define OPTIMIZE_DELAY 5
479
480/*
481 * Optimize (replace a breakpoint with a jump) kprobes listed on
482 * optimizing_list.
483 */
484static void do_optimize_kprobes(void)
485{
486 /*
487 * The optimization/unoptimization refers online_cpus via
488 * stop_machine() and cpu-hotplug modifies online_cpus.
489 * And same time, text_mutex will be held in cpu-hotplug and here.
490 * This combination can cause a deadlock (cpu-hotplug try to lock
491 * text_mutex but stop_machine can not be done because online_cpus
492 * has been changed)
493 * To avoid this deadlock, caller must have locked cpu hotplug
494 * for preventing cpu-hotplug outside of text_mutex locking.
495 */
496 lockdep_assert_cpus_held();
497
498 /* Optimization never be done when disarmed */
499 if (kprobes_all_disarmed || !kprobes_allow_optimization ||
500 list_empty(&optimizing_list))
501 return;
502
503 mutex_lock(&text_mutex);
504 arch_optimize_kprobes(&optimizing_list);
505 mutex_unlock(&text_mutex);
506}
507
508/*
509 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
510 * if need) kprobes listed on unoptimizing_list.
511 */
512static void do_unoptimize_kprobes(void)
513{
514 struct optimized_kprobe *op, *tmp;
515
516 /* See comment in do_optimize_kprobes() */
517 lockdep_assert_cpus_held();
518
519 /* Unoptimization must be done anytime */
520 if (list_empty(&unoptimizing_list))
521 return;
522
523 mutex_lock(&text_mutex);
524 arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
525 /* Loop free_list for disarming */
526 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
527 /* Disarm probes if marked disabled */
528 if (kprobe_disabled(&op->kp))
529 arch_disarm_kprobe(&op->kp);
530 if (kprobe_unused(&op->kp)) {
531 /*
532 * Remove unused probes from hash list. After waiting
533 * for synchronization, these probes are reclaimed.
534 * (reclaiming is done by do_free_cleaned_kprobes.)
535 */
536 hlist_del_rcu(&op->kp.hlist);
537 } else
538 list_del_init(&op->list);
539 }
540 mutex_unlock(&text_mutex);
541}
542
543/* Reclaim all kprobes on the free_list */
544static void do_free_cleaned_kprobes(void)
545{
546 struct optimized_kprobe *op, *tmp;
547
548 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
549 BUG_ON(!kprobe_unused(&op->kp));
550 list_del_init(&op->list);
551 free_aggr_kprobe(&op->kp);
552 }
553}
554
555/* Start optimizer after OPTIMIZE_DELAY passed */
556static void kick_kprobe_optimizer(void)
557{
558 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
559}
560
561/* Kprobe jump optimizer */
562static void kprobe_optimizer(struct work_struct *work)
563{
564 mutex_lock(&kprobe_mutex);
565 cpus_read_lock();
566 /* Lock modules while optimizing kprobes */
567 mutex_lock(&module_mutex);
568
569 /*
570 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
571 * kprobes before waiting for quiesence period.
572 */
573 do_unoptimize_kprobes();
574
575 /*
576 * Step 2: Wait for quiesence period to ensure all potentially
577 * preempted tasks to have normally scheduled. Because optprobe
578 * may modify multiple instructions, there is a chance that Nth
579 * instruction is preempted. In that case, such tasks can return
580 * to 2nd-Nth byte of jump instruction. This wait is for avoiding it.
581 * Note that on non-preemptive kernel, this is transparently converted
582 * to synchronoze_sched() to wait for all interrupts to have completed.
583 */
584 synchronize_rcu_tasks();
585
586 /* Step 3: Optimize kprobes after quiesence period */
587 do_optimize_kprobes();
588
589 /* Step 4: Free cleaned kprobes after quiesence period */
590 do_free_cleaned_kprobes();
591
592 mutex_unlock(&module_mutex);
593 cpus_read_unlock();
594 mutex_unlock(&kprobe_mutex);
595
596 /* Step 5: Kick optimizer again if needed */
597 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
598 kick_kprobe_optimizer();
599}
600
601/* Wait for completing optimization and unoptimization */
602void wait_for_kprobe_optimizer(void)
603{
604 mutex_lock(&kprobe_mutex);
605
606 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
607 mutex_unlock(&kprobe_mutex);
608
609 /* this will also make optimizing_work execute immmediately */
610 flush_delayed_work(&optimizing_work);
611 /* @optimizing_work might not have been queued yet, relax */
612 cpu_relax();
613
614 mutex_lock(&kprobe_mutex);
615 }
616
617 mutex_unlock(&kprobe_mutex);
618}
619
620/* Optimize kprobe if p is ready to be optimized */
621static void optimize_kprobe(struct kprobe *p)
622{
623 struct optimized_kprobe *op;
624
625 /* Check if the kprobe is disabled or not ready for optimization. */
626 if (!kprobe_optready(p) || !kprobes_allow_optimization ||
627 (kprobe_disabled(p) || kprobes_all_disarmed))
628 return;
629
630 /* kprobes with post_handler can not be optimized */
631 if (p->post_handler)
632 return;
633
634 op = container_of(p, struct optimized_kprobe, kp);
635
636 /* Check there is no other kprobes at the optimized instructions */
637 if (arch_check_optimized_kprobe(op) < 0)
638 return;
639
640 /* Check if it is already optimized. */
641 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
642 return;
643 op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
644
645 if (!list_empty(&op->list))
646 /* This is under unoptimizing. Just dequeue the probe */
647 list_del_init(&op->list);
648 else {
649 list_add(&op->list, &optimizing_list);
650 kick_kprobe_optimizer();
651 }
652}
653
654/* Short cut to direct unoptimizing */
655static void force_unoptimize_kprobe(struct optimized_kprobe *op)
656{
657 lockdep_assert_cpus_held();
658 arch_unoptimize_kprobe(op);
659 if (kprobe_disabled(&op->kp))
660 arch_disarm_kprobe(&op->kp);
661}
662
663/* Unoptimize a kprobe if p is optimized */
664static void unoptimize_kprobe(struct kprobe *p, bool force)
665{
666 struct optimized_kprobe *op;
667
668 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
669 return; /* This is not an optprobe nor optimized */
670
671 op = container_of(p, struct optimized_kprobe, kp);
672 if (!kprobe_optimized(p)) {
673 /* Unoptimized or unoptimizing case */
674 if (force && !list_empty(&op->list)) {
675 /*
676 * Only if this is unoptimizing kprobe and forced,
677 * forcibly unoptimize it. (No need to unoptimize
678 * unoptimized kprobe again :)
679 */
680 list_del_init(&op->list);
681 force_unoptimize_kprobe(op);
682 }
683 return;
684 }
685
686 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
687 if (!list_empty(&op->list)) {
688 /* Dequeue from the optimization queue */
689 list_del_init(&op->list);
690 return;
691 }
692 /* Optimized kprobe case */
693 if (force)
694 /* Forcibly update the code: this is a special case */
695 force_unoptimize_kprobe(op);
696 else {
697 list_add(&op->list, &unoptimizing_list);
698 kick_kprobe_optimizer();
699 }
700}
701
702/* Cancel unoptimizing for reusing */
703static void reuse_unused_kprobe(struct kprobe *ap)
704{
705 struct optimized_kprobe *op;
706
707 BUG_ON(!kprobe_unused(ap));
708 /*
709 * Unused kprobe MUST be on the way of delayed unoptimizing (means
710 * there is still a relative jump) and disabled.
711 */
712 op = container_of(ap, struct optimized_kprobe, kp);
713 WARN_ON_ONCE(list_empty(&op->list));
714 /* Enable the probe again */
715 ap->flags &= ~KPROBE_FLAG_DISABLED;
716 /* Optimize it again (remove from op->list) */
717 BUG_ON(!kprobe_optready(ap));
718 optimize_kprobe(ap);
719}
720
721/* Remove optimized instructions */
722static void kill_optimized_kprobe(struct kprobe *p)
723{
724 struct optimized_kprobe *op;
725
726 op = container_of(p, struct optimized_kprobe, kp);
727 if (!list_empty(&op->list))
728 /* Dequeue from the (un)optimization queue */
729 list_del_init(&op->list);
730 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
731
732 if (kprobe_unused(p)) {
733 /* Enqueue if it is unused */
734 list_add(&op->list, &freeing_list);
735 /*
736 * Remove unused probes from the hash list. After waiting
737 * for synchronization, this probe is reclaimed.
738 * (reclaiming is done by do_free_cleaned_kprobes().)
739 */
740 hlist_del_rcu(&op->kp.hlist);
741 }
742
743 /* Don't touch the code, because it is already freed. */
744 arch_remove_optimized_kprobe(op);
745}
746
747static inline
748void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
749{
750 if (!kprobe_ftrace(p))
751 arch_prepare_optimized_kprobe(op, p);
752}
753
754/* Try to prepare optimized instructions */
755static void prepare_optimized_kprobe(struct kprobe *p)
756{
757 struct optimized_kprobe *op;
758
759 op = container_of(p, struct optimized_kprobe, kp);
760 __prepare_optimized_kprobe(op, p);
761}
762
763/* Allocate new optimized_kprobe and try to prepare optimized instructions */
764static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
765{
766 struct optimized_kprobe *op;
767
768 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
769 if (!op)
770 return NULL;
771
772 INIT_LIST_HEAD(&op->list);
773 op->kp.addr = p->addr;
774 __prepare_optimized_kprobe(op, p);
775
776 return &op->kp;
777}
778
779static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
780
781/*
782 * Prepare an optimized_kprobe and optimize it
783 * NOTE: p must be a normal registered kprobe
784 */
785static void try_to_optimize_kprobe(struct kprobe *p)
786{
787 struct kprobe *ap;
788 struct optimized_kprobe *op;
789
790 /* Impossible to optimize ftrace-based kprobe */
791 if (kprobe_ftrace(p))
792 return;
793
794 /* For preparing optimization, jump_label_text_reserved() is called */
795 cpus_read_lock();
796 jump_label_lock();
797 mutex_lock(&text_mutex);
798
799 ap = alloc_aggr_kprobe(p);
800 if (!ap)
801 goto out;
802
803 op = container_of(ap, struct optimized_kprobe, kp);
804 if (!arch_prepared_optinsn(&op->optinsn)) {
805 /* If failed to setup optimizing, fallback to kprobe */
806 arch_remove_optimized_kprobe(op);
807 kfree(op);
808 goto out;
809 }
810
811 init_aggr_kprobe(ap, p);
812 optimize_kprobe(ap); /* This just kicks optimizer thread */
813
814out:
815 mutex_unlock(&text_mutex);
816 jump_label_unlock();
817 cpus_read_unlock();
818}
819
820#ifdef CONFIG_SYSCTL
821static void optimize_all_kprobes(void)
822{
823 struct hlist_head *head;
824 struct kprobe *p;
825 unsigned int i;
826
827 mutex_lock(&kprobe_mutex);
828 /* If optimization is already allowed, just return */
829 if (kprobes_allow_optimization)
830 goto out;
831
832 cpus_read_lock();
833 kprobes_allow_optimization = true;
834 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
835 head = &kprobe_table[i];
836 hlist_for_each_entry_rcu(p, head, hlist)
837 if (!kprobe_disabled(p))
838 optimize_kprobe(p);
839 }
840 cpus_read_unlock();
841 printk(KERN_INFO "Kprobes globally optimized\n");
842out:
843 mutex_unlock(&kprobe_mutex);
844}
845
846static void unoptimize_all_kprobes(void)
847{
848 struct hlist_head *head;
849 struct kprobe *p;
850 unsigned int i;
851
852 mutex_lock(&kprobe_mutex);
853 /* If optimization is already prohibited, just return */
854 if (!kprobes_allow_optimization) {
855 mutex_unlock(&kprobe_mutex);
856 return;
857 }
858
859 cpus_read_lock();
860 kprobes_allow_optimization = false;
861 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
862 head = &kprobe_table[i];
863 hlist_for_each_entry_rcu(p, head, hlist) {
864 if (!kprobe_disabled(p))
865 unoptimize_kprobe(p, false);
866 }
867 }
868 cpus_read_unlock();
869 mutex_unlock(&kprobe_mutex);
870
871 /* Wait for unoptimizing completion */
872 wait_for_kprobe_optimizer();
873 printk(KERN_INFO "Kprobes globally unoptimized\n");
874}
875
876static DEFINE_MUTEX(kprobe_sysctl_mutex);
877int sysctl_kprobes_optimization;
878int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
879 void __user *buffer, size_t *length,
880 loff_t *ppos)
881{
882 int ret;
883
884 mutex_lock(&kprobe_sysctl_mutex);
885 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
886 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
887
888 if (sysctl_kprobes_optimization)
889 optimize_all_kprobes();
890 else
891 unoptimize_all_kprobes();
892 mutex_unlock(&kprobe_sysctl_mutex);
893
894 return ret;
895}
896#endif /* CONFIG_SYSCTL */
897
898/* Put a breakpoint for a probe. Must be called with text_mutex locked */
899static void __arm_kprobe(struct kprobe *p)
900{
901 struct kprobe *_p;
902
903 /* Check collision with other optimized kprobes */
904 _p = get_optimized_kprobe((unsigned long)p->addr);
905 if (unlikely(_p))
906 /* Fallback to unoptimized kprobe */
907 unoptimize_kprobe(_p, true);
908
909 arch_arm_kprobe(p);
910 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */
911}
912
913/* Remove the breakpoint of a probe. Must be called with text_mutex locked */
914static void __disarm_kprobe(struct kprobe *p, bool reopt)
915{
916 struct kprobe *_p;
917
918 /* Try to unoptimize */
919 unoptimize_kprobe(p, kprobes_all_disarmed);
920
921 if (!kprobe_queued(p)) {
922 arch_disarm_kprobe(p);
923 /* If another kprobe was blocked, optimize it. */
924 _p = get_optimized_kprobe((unsigned long)p->addr);
925 if (unlikely(_p) && reopt)
926 optimize_kprobe(_p);
927 }
928 /* TODO: reoptimize others after unoptimized this probe */
929}
930
931#else /* !CONFIG_OPTPROBES */
932
933#define optimize_kprobe(p) do {} while (0)
934#define unoptimize_kprobe(p, f) do {} while (0)
935#define kill_optimized_kprobe(p) do {} while (0)
936#define prepare_optimized_kprobe(p) do {} while (0)
937#define try_to_optimize_kprobe(p) do {} while (0)
938#define __arm_kprobe(p) arch_arm_kprobe(p)
939#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
940#define kprobe_disarmed(p) kprobe_disabled(p)
941#define wait_for_kprobe_optimizer() do {} while (0)
942
943/* There should be no unused kprobes can be reused without optimization */
944static void reuse_unused_kprobe(struct kprobe *ap)
945{
946 printk(KERN_ERR "Error: There should be no unused kprobe here.\n");
947 BUG_ON(kprobe_unused(ap));
948}
949
950static void free_aggr_kprobe(struct kprobe *p)
951{
952 arch_remove_kprobe(p);
953 kfree(p);
954}
955
956static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
957{
958 return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
959}
960#endif /* CONFIG_OPTPROBES */
961
962#ifdef CONFIG_KPROBES_ON_FTRACE
963static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
964 .func = kprobe_ftrace_handler,
965 .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
966};
967static int kprobe_ftrace_enabled;
968
969/* Must ensure p->addr is really on ftrace */
970static int prepare_kprobe(struct kprobe *p)
971{
972 if (!kprobe_ftrace(p))
973 return arch_prepare_kprobe(p);
974
975 return arch_prepare_kprobe_ftrace(p);
976}
977
978/* Caller must lock kprobe_mutex */
979static int arm_kprobe_ftrace(struct kprobe *p)
980{
981 int ret = 0;
982
983 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
984 (unsigned long)p->addr, 0, 0);
985 if (ret) {
986 pr_debug("Failed to arm kprobe-ftrace at %pS (%d)\n",
987 p->addr, ret);
988 return ret;
989 }
990
991 if (kprobe_ftrace_enabled == 0) {
992 ret = register_ftrace_function(&kprobe_ftrace_ops);
993 if (ret) {
994 pr_debug("Failed to init kprobe-ftrace (%d)\n", ret);
995 goto err_ftrace;
996 }
997 }
998
999 kprobe_ftrace_enabled++;
1000 return ret;
1001
1002err_ftrace:
1003 /*
1004 * Note: Since kprobe_ftrace_ops has IPMODIFY set, and ftrace requires a
1005 * non-empty filter_hash for IPMODIFY ops, we're safe from an accidental
1006 * empty filter_hash which would undesirably trace all functions.
1007 */
1008 ftrace_set_filter_ip(&kprobe_ftrace_ops, (unsigned long)p->addr, 1, 0);
1009 return ret;
1010}
1011
1012/* Caller must lock kprobe_mutex */
1013static int disarm_kprobe_ftrace(struct kprobe *p)
1014{
1015 int ret = 0;
1016
1017 if (kprobe_ftrace_enabled == 1) {
1018 ret = unregister_ftrace_function(&kprobe_ftrace_ops);
1019 if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (%d)\n", ret))
1020 return ret;
1021 }
1022
1023 kprobe_ftrace_enabled--;
1024
1025 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
1026 (unsigned long)p->addr, 1, 0);
1027 WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (%d)\n",
1028 p->addr, ret);
1029 return ret;
1030}
1031#else /* !CONFIG_KPROBES_ON_FTRACE */
1032#define prepare_kprobe(p) arch_prepare_kprobe(p)
1033#define arm_kprobe_ftrace(p) (-ENODEV)
1034#define disarm_kprobe_ftrace(p) (-ENODEV)
1035#endif
1036
1037/* Arm a kprobe with text_mutex */
1038static int arm_kprobe(struct kprobe *kp)
1039{
1040 if (unlikely(kprobe_ftrace(kp)))
1041 return arm_kprobe_ftrace(kp);
1042
1043 cpus_read_lock();
1044 mutex_lock(&text_mutex);
1045 __arm_kprobe(kp);
1046 mutex_unlock(&text_mutex);
1047 cpus_read_unlock();
1048
1049 return 0;
1050}
1051
1052/* Disarm a kprobe with text_mutex */
1053static int disarm_kprobe(struct kprobe *kp, bool reopt)
1054{
1055 if (unlikely(kprobe_ftrace(kp)))
1056 return disarm_kprobe_ftrace(kp);
1057
1058 cpus_read_lock();
1059 mutex_lock(&text_mutex);
1060 __disarm_kprobe(kp, reopt);
1061 mutex_unlock(&text_mutex);
1062 cpus_read_unlock();
1063
1064 return 0;
1065}
1066
1067/*
1068 * Aggregate handlers for multiple kprobes support - these handlers
1069 * take care of invoking the individual kprobe handlers on p->list
1070 */
1071static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
1072{
1073 struct kprobe *kp;
1074
1075 list_for_each_entry_rcu(kp, &p->list, list) {
1076 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
1077 set_kprobe_instance(kp);
1078 if (kp->pre_handler(kp, regs))
1079 return 1;
1080 }
1081 reset_kprobe_instance();
1082 }
1083 return 0;
1084}
1085NOKPROBE_SYMBOL(aggr_pre_handler);
1086
1087static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1088 unsigned long flags)
1089{
1090 struct kprobe *kp;
1091
1092 list_for_each_entry_rcu(kp, &p->list, list) {
1093 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
1094 set_kprobe_instance(kp);
1095 kp->post_handler(kp, regs, flags);
1096 reset_kprobe_instance();
1097 }
1098 }
1099}
1100NOKPROBE_SYMBOL(aggr_post_handler);
1101
1102static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
1103 int trapnr)
1104{
1105 struct kprobe *cur = __this_cpu_read(kprobe_instance);
1106
1107 /*
1108 * if we faulted "during" the execution of a user specified
1109 * probe handler, invoke just that probe's fault handler
1110 */
1111 if (cur && cur->fault_handler) {
1112 if (cur->fault_handler(cur, regs, trapnr))
1113 return 1;
1114 }
1115 return 0;
1116}
1117NOKPROBE_SYMBOL(aggr_fault_handler);
1118
1119/* Walks the list and increments nmissed count for multiprobe case */
1120void kprobes_inc_nmissed_count(struct kprobe *p)
1121{
1122 struct kprobe *kp;
1123 if (!kprobe_aggrprobe(p)) {
1124 p->nmissed++;
1125 } else {
1126 list_for_each_entry_rcu(kp, &p->list, list)
1127 kp->nmissed++;
1128 }
1129 return;
1130}
1131NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1132
1133void recycle_rp_inst(struct kretprobe_instance *ri,
1134 struct hlist_head *head)
1135{
1136 struct kretprobe *rp = ri->rp;
1137
1138 /* remove rp inst off the rprobe_inst_table */
1139 hlist_del(&ri->hlist);
1140 INIT_HLIST_NODE(&ri->hlist);
1141 if (likely(rp)) {
1142 raw_spin_lock(&rp->lock);
1143 hlist_add_head(&ri->hlist, &rp->free_instances);
1144 raw_spin_unlock(&rp->lock);
1145 } else
1146 /* Unregistering */
1147 hlist_add_head(&ri->hlist, head);
1148}
1149NOKPROBE_SYMBOL(recycle_rp_inst);
1150
1151void kretprobe_hash_lock(struct task_struct *tsk,
1152 struct hlist_head **head, unsigned long *flags)
1153__acquires(hlist_lock)
1154{
1155 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
1156 raw_spinlock_t *hlist_lock;
1157
1158 *head = &kretprobe_inst_table[hash];
1159 hlist_lock = kretprobe_table_lock_ptr(hash);
1160 raw_spin_lock_irqsave(hlist_lock, *flags);
1161}
1162NOKPROBE_SYMBOL(kretprobe_hash_lock);
1163
1164static void kretprobe_table_lock(unsigned long hash,
1165 unsigned long *flags)
1166__acquires(hlist_lock)
1167{
1168 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1169 raw_spin_lock_irqsave(hlist_lock, *flags);
1170}
1171NOKPROBE_SYMBOL(kretprobe_table_lock);
1172
1173void kretprobe_hash_unlock(struct task_struct *tsk,
1174 unsigned long *flags)
1175__releases(hlist_lock)
1176{
1177 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
1178 raw_spinlock_t *hlist_lock;
1179
1180 hlist_lock = kretprobe_table_lock_ptr(hash);
1181 raw_spin_unlock_irqrestore(hlist_lock, *flags);
1182}
1183NOKPROBE_SYMBOL(kretprobe_hash_unlock);
1184
1185static void kretprobe_table_unlock(unsigned long hash,
1186 unsigned long *flags)
1187__releases(hlist_lock)
1188{
1189 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1190 raw_spin_unlock_irqrestore(hlist_lock, *flags);
1191}
1192NOKPROBE_SYMBOL(kretprobe_table_unlock);
1193
1194/*
1195 * This function is called from finish_task_switch when task tk becomes dead,
1196 * so that we can recycle any function-return probe instances associated
1197 * with this task. These left over instances represent probed functions
1198 * that have been called but will never return.
1199 */
1200void kprobe_flush_task(struct task_struct *tk)
1201{
1202 struct kretprobe_instance *ri;
1203 struct hlist_head *head, empty_rp;
1204 struct hlist_node *tmp;
1205 unsigned long hash, flags = 0;
1206
1207 if (unlikely(!kprobes_initialized))
1208 /* Early boot. kretprobe_table_locks not yet initialized. */
1209 return;
1210
1211 INIT_HLIST_HEAD(&empty_rp);
1212 hash = hash_ptr(tk, KPROBE_HASH_BITS);
1213 head = &kretprobe_inst_table[hash];
1214 kretprobe_table_lock(hash, &flags);
1215 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
1216 if (ri->task == tk)
1217 recycle_rp_inst(ri, &empty_rp);
1218 }
1219 kretprobe_table_unlock(hash, &flags);
1220 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
1221 hlist_del(&ri->hlist);
1222 kfree(ri);
1223 }
1224}
1225NOKPROBE_SYMBOL(kprobe_flush_task);
1226
1227static inline void free_rp_inst(struct kretprobe *rp)
1228{
1229 struct kretprobe_instance *ri;
1230 struct hlist_node *next;
1231
1232 hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) {
1233 hlist_del(&ri->hlist);
1234 kfree(ri);
1235 }
1236}
1237
1238static void cleanup_rp_inst(struct kretprobe *rp)
1239{
1240 unsigned long flags, hash;
1241 struct kretprobe_instance *ri;
1242 struct hlist_node *next;
1243 struct hlist_head *head;
1244
1245 /* No race here */
1246 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
1247 kretprobe_table_lock(hash, &flags);
1248 head = &kretprobe_inst_table[hash];
1249 hlist_for_each_entry_safe(ri, next, head, hlist) {
1250 if (ri->rp == rp)
1251 ri->rp = NULL;
1252 }
1253 kretprobe_table_unlock(hash, &flags);
1254 }
1255 free_rp_inst(rp);
1256}
1257NOKPROBE_SYMBOL(cleanup_rp_inst);
1258
1259/* Add the new probe to ap->list */
1260static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)
1261{
1262 BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
1263
1264 if (p->post_handler)
1265 unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */
1266
1267 list_add_rcu(&p->list, &ap->list);
1268 if (p->post_handler && !ap->post_handler)
1269 ap->post_handler = aggr_post_handler;
1270
1271 return 0;
1272}
1273
1274/*
1275 * Fill in the required fields of the "manager kprobe". Replace the
1276 * earlier kprobe in the hlist with the manager kprobe
1277 */
1278static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
1279{
1280 /* Copy p's insn slot to ap */
1281 copy_kprobe(p, ap);
1282 flush_insn_slot(ap);
1283 ap->addr = p->addr;
1284 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1285 ap->pre_handler = aggr_pre_handler;
1286 ap->fault_handler = aggr_fault_handler;
1287 /* We don't care the kprobe which has gone. */
1288 if (p->post_handler && !kprobe_gone(p))
1289 ap->post_handler = aggr_post_handler;
1290
1291 INIT_LIST_HEAD(&ap->list);
1292 INIT_HLIST_NODE(&ap->hlist);
1293
1294 list_add_rcu(&p->list, &ap->list);
1295 hlist_replace_rcu(&p->hlist, &ap->hlist);
1296}
1297
1298/*
1299 * This is the second or subsequent kprobe at the address - handle
1300 * the intricacies
1301 */
1302static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p)
1303{
1304 int ret = 0;
1305 struct kprobe *ap = orig_p;
1306
1307 cpus_read_lock();
1308
1309 /* For preparing optimization, jump_label_text_reserved() is called */
1310 jump_label_lock();
1311 mutex_lock(&text_mutex);
1312
1313 if (!kprobe_aggrprobe(orig_p)) {
1314 /* If orig_p is not an aggr_kprobe, create new aggr_kprobe. */
1315 ap = alloc_aggr_kprobe(orig_p);
1316 if (!ap) {
1317 ret = -ENOMEM;
1318 goto out;
1319 }
1320 init_aggr_kprobe(ap, orig_p);
1321 } else if (kprobe_unused(ap))
1322 /* This probe is going to die. Rescue it */
1323 reuse_unused_kprobe(ap);
1324
1325 if (kprobe_gone(ap)) {
1326 /*
1327 * Attempting to insert new probe at the same location that
1328 * had a probe in the module vaddr area which already
1329 * freed. So, the instruction slot has already been
1330 * released. We need a new slot for the new probe.
1331 */
1332 ret = arch_prepare_kprobe(ap);
1333 if (ret)
1334 /*
1335 * Even if fail to allocate new slot, don't need to
1336 * free aggr_probe. It will be used next time, or
1337 * freed by unregister_kprobe.
1338 */
1339 goto out;
1340
1341 /* Prepare optimized instructions if possible. */
1342 prepare_optimized_kprobe(ap);
1343
1344 /*
1345 * Clear gone flag to prevent allocating new slot again, and
1346 * set disabled flag because it is not armed yet.
1347 */
1348 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1349 | KPROBE_FLAG_DISABLED;
1350 }
1351
1352 /* Copy ap's insn slot to p */
1353 copy_kprobe(ap, p);
1354 ret = add_new_kprobe(ap, p);
1355
1356out:
1357 mutex_unlock(&text_mutex);
1358 jump_label_unlock();
1359 cpus_read_unlock();
1360
1361 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1362 ap->flags &= ~KPROBE_FLAG_DISABLED;
1363 if (!kprobes_all_disarmed) {
1364 /* Arm the breakpoint again. */
1365 ret = arm_kprobe(ap);
1366 if (ret) {
1367 ap->flags |= KPROBE_FLAG_DISABLED;
1368 list_del_rcu(&p->list);
1369 synchronize_sched();
1370 }
1371 }
1372 }
1373 return ret;
1374}
1375
1376bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1377{
1378 /* The __kprobes marked functions and entry code must not be probed */
1379 return addr >= (unsigned long)__kprobes_text_start &&
1380 addr < (unsigned long)__kprobes_text_end;
1381}
1382
1383bool within_kprobe_blacklist(unsigned long addr)
1384{
1385 struct kprobe_blacklist_entry *ent;
1386
1387 if (arch_within_kprobe_blacklist(addr))
1388 return true;
1389 /*
1390 * If there exists a kprobe_blacklist, verify and
1391 * fail any probe registration in the prohibited area
1392 */
1393 list_for_each_entry(ent, &kprobe_blacklist, list) {
1394 if (addr >= ent->start_addr && addr < ent->end_addr)
1395 return true;
1396 }
1397
1398 return false;
1399}
1400
1401/*
1402 * If we have a symbol_name argument, look it up and add the offset field
1403 * to it. This way, we can specify a relative address to a symbol.
1404 * This returns encoded errors if it fails to look up symbol or invalid
1405 * combination of parameters.
1406 */
1407static kprobe_opcode_t *_kprobe_addr(kprobe_opcode_t *addr,
1408 const char *symbol_name, unsigned int offset)
1409{
1410 if ((symbol_name && addr) || (!symbol_name && !addr))
1411 goto invalid;
1412
1413 if (symbol_name) {
1414 addr = kprobe_lookup_name(symbol_name, offset);
1415 if (!addr)
1416 return ERR_PTR(-ENOENT);
1417 }
1418
1419 addr = (kprobe_opcode_t *)(((char *)addr) + offset);
1420 if (addr)
1421 return addr;
1422
1423invalid:
1424 return ERR_PTR(-EINVAL);
1425}
1426
1427static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
1428{
1429 return _kprobe_addr(p->addr, p->symbol_name, p->offset);
1430}
1431
1432/* Check passed kprobe is valid and return kprobe in kprobe_table. */
1433static struct kprobe *__get_valid_kprobe(struct kprobe *p)
1434{
1435 struct kprobe *ap, *list_p;
1436
1437 ap = get_kprobe(p->addr);
1438 if (unlikely(!ap))
1439 return NULL;
1440
1441 if (p != ap) {
1442 list_for_each_entry_rcu(list_p, &ap->list, list)
1443 if (list_p == p)
1444 /* kprobe p is a valid probe */
1445 goto valid;
1446 return NULL;
1447 }
1448valid:
1449 return ap;
1450}
1451
1452/* Return error if the kprobe is being re-registered */
1453static inline int check_kprobe_rereg(struct kprobe *p)
1454{
1455 int ret = 0;
1456
1457 mutex_lock(&kprobe_mutex);
1458 if (__get_valid_kprobe(p))
1459 ret = -EINVAL;
1460 mutex_unlock(&kprobe_mutex);
1461
1462 return ret;
1463}
1464
1465int __weak arch_check_ftrace_location(struct kprobe *p)
1466{
1467 unsigned long ftrace_addr;
1468
1469 ftrace_addr = ftrace_location((unsigned long)p->addr);
1470 if (ftrace_addr) {
1471#ifdef CONFIG_KPROBES_ON_FTRACE
1472 /* Given address is not on the instruction boundary */
1473 if ((unsigned long)p->addr != ftrace_addr)
1474 return -EILSEQ;
1475 p->flags |= KPROBE_FLAG_FTRACE;
1476#else /* !CONFIG_KPROBES_ON_FTRACE */
1477 return -EINVAL;
1478#endif
1479 }
1480 return 0;
1481}
1482
1483static int check_kprobe_address_safe(struct kprobe *p,
1484 struct module **probed_mod)
1485{
1486 int ret;
1487
1488 ret = arch_check_ftrace_location(p);
1489 if (ret)
1490 return ret;
1491 jump_label_lock();
1492 preempt_disable();
1493
1494 /* Ensure it is not in reserved area nor out of text */
1495 if (!kernel_text_address((unsigned long) p->addr) ||
1496 within_kprobe_blacklist((unsigned long) p->addr) ||
1497 jump_label_text_reserved(p->addr, p->addr)) {
1498 ret = -EINVAL;
1499 goto out;
1500 }
1501
1502 /* Check if are we probing a module */
1503 *probed_mod = __module_text_address((unsigned long) p->addr);
1504 if (*probed_mod) {
1505 /*
1506 * We must hold a refcount of the probed module while updating
1507 * its code to prohibit unexpected unloading.
1508 */
1509 if (unlikely(!try_module_get(*probed_mod))) {
1510 ret = -ENOENT;
1511 goto out;
1512 }
1513
1514 /*
1515 * If the module freed .init.text, we couldn't insert
1516 * kprobes in there.
1517 */
1518 if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1519 (*probed_mod)->state != MODULE_STATE_COMING) {
1520 module_put(*probed_mod);
1521 *probed_mod = NULL;
1522 ret = -ENOENT;
1523 }
1524 }
1525out:
1526 preempt_enable();
1527 jump_label_unlock();
1528
1529 return ret;
1530}
1531
1532int register_kprobe(struct kprobe *p)
1533{
1534 int ret;
1535 struct kprobe *old_p;
1536 struct module *probed_mod;
1537 kprobe_opcode_t *addr;
1538
1539 /* Adjust probe address from symbol */
1540 addr = kprobe_addr(p);
1541 if (IS_ERR(addr))
1542 return PTR_ERR(addr);
1543 p->addr = addr;
1544
1545 ret = check_kprobe_rereg(p);
1546 if (ret)
1547 return ret;
1548
1549 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1550 p->flags &= KPROBE_FLAG_DISABLED;
1551 p->nmissed = 0;
1552 INIT_LIST_HEAD(&p->list);
1553
1554 ret = check_kprobe_address_safe(p, &probed_mod);
1555 if (ret)
1556 return ret;
1557
1558 mutex_lock(&kprobe_mutex);
1559
1560 old_p = get_kprobe(p->addr);
1561 if (old_p) {
1562 /* Since this may unoptimize old_p, locking text_mutex. */
1563 ret = register_aggr_kprobe(old_p, p);
1564 goto out;
1565 }
1566
1567 cpus_read_lock();
1568 /* Prevent text modification */
1569 mutex_lock(&text_mutex);
1570 ret = prepare_kprobe(p);
1571 mutex_unlock(&text_mutex);
1572 cpus_read_unlock();
1573 if (ret)
1574 goto out;
1575
1576 INIT_HLIST_NODE(&p->hlist);
1577 hlist_add_head_rcu(&p->hlist,
1578 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1579
1580 if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
1581 ret = arm_kprobe(p);
1582 if (ret) {
1583 hlist_del_rcu(&p->hlist);
1584 synchronize_sched();
1585 goto out;
1586 }
1587 }
1588
1589 /* Try to optimize kprobe */
1590 try_to_optimize_kprobe(p);
1591out:
1592 mutex_unlock(&kprobe_mutex);
1593
1594 if (probed_mod)
1595 module_put(probed_mod);
1596
1597 return ret;
1598}
1599EXPORT_SYMBOL_GPL(register_kprobe);
1600
1601/* Check if all probes on the aggrprobe are disabled */
1602static int aggr_kprobe_disabled(struct kprobe *ap)
1603{
1604 struct kprobe *kp;
1605
1606 list_for_each_entry_rcu(kp, &ap->list, list)
1607 if (!kprobe_disabled(kp))
1608 /*
1609 * There is an active probe on the list.
1610 * We can't disable this ap.
1611 */
1612 return 0;
1613
1614 return 1;
1615}
1616
1617/* Disable one kprobe: Make sure called under kprobe_mutex is locked */
1618static struct kprobe *__disable_kprobe(struct kprobe *p)
1619{
1620 struct kprobe *orig_p;
1621 int ret;
1622
1623 /* Get an original kprobe for return */
1624 orig_p = __get_valid_kprobe(p);
1625 if (unlikely(orig_p == NULL))
1626 return ERR_PTR(-EINVAL);
1627
1628 if (!kprobe_disabled(p)) {
1629 /* Disable probe if it is a child probe */
1630 if (p != orig_p)
1631 p->flags |= KPROBE_FLAG_DISABLED;
1632
1633 /* Try to disarm and disable this/parent probe */
1634 if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
1635 /*
1636 * If kprobes_all_disarmed is set, orig_p
1637 * should have already been disarmed, so
1638 * skip unneed disarming process.
1639 */
1640 if (!kprobes_all_disarmed) {
1641 ret = disarm_kprobe(orig_p, true);
1642 if (ret) {
1643 p->flags &= ~KPROBE_FLAG_DISABLED;
1644 return ERR_PTR(ret);
1645 }
1646 }
1647 orig_p->flags |= KPROBE_FLAG_DISABLED;
1648 }
1649 }
1650
1651 return orig_p;
1652}
1653
1654/*
1655 * Unregister a kprobe without a scheduler synchronization.
1656 */
1657static int __unregister_kprobe_top(struct kprobe *p)
1658{
1659 struct kprobe *ap, *list_p;
1660
1661 /* Disable kprobe. This will disarm it if needed. */
1662 ap = __disable_kprobe(p);
1663 if (IS_ERR(ap))
1664 return PTR_ERR(ap);
1665
1666 if (ap == p)
1667 /*
1668 * This probe is an independent(and non-optimized) kprobe
1669 * (not an aggrprobe). Remove from the hash list.
1670 */
1671 goto disarmed;
1672
1673 /* Following process expects this probe is an aggrprobe */
1674 WARN_ON(!kprobe_aggrprobe(ap));
1675
1676 if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1677 /*
1678 * !disarmed could be happen if the probe is under delayed
1679 * unoptimizing.
1680 */
1681 goto disarmed;
1682 else {
1683 /* If disabling probe has special handlers, update aggrprobe */
1684 if (p->post_handler && !kprobe_gone(p)) {
1685 list_for_each_entry_rcu(list_p, &ap->list, list) {
1686 if ((list_p != p) && (list_p->post_handler))
1687 goto noclean;
1688 }
1689 ap->post_handler = NULL;
1690 }
1691noclean:
1692 /*
1693 * Remove from the aggrprobe: this path will do nothing in
1694 * __unregister_kprobe_bottom().
1695 */
1696 list_del_rcu(&p->list);
1697 if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1698 /*
1699 * Try to optimize this probe again, because post
1700 * handler may have been changed.
1701 */
1702 optimize_kprobe(ap);
1703 }
1704 return 0;
1705
1706disarmed:
1707 BUG_ON(!kprobe_disarmed(ap));
1708 hlist_del_rcu(&ap->hlist);
1709 return 0;
1710}
1711
1712static void __unregister_kprobe_bottom(struct kprobe *p)
1713{
1714 struct kprobe *ap;
1715
1716 if (list_empty(&p->list))
1717 /* This is an independent kprobe */
1718 arch_remove_kprobe(p);
1719 else if (list_is_singular(&p->list)) {
1720 /* This is the last child of an aggrprobe */
1721 ap = list_entry(p->list.next, struct kprobe, list);
1722 list_del(&p->list);
1723 free_aggr_kprobe(ap);
1724 }
1725 /* Otherwise, do nothing. */
1726}
1727
1728int register_kprobes(struct kprobe **kps, int num)
1729{
1730 int i, ret = 0;
1731
1732 if (num <= 0)
1733 return -EINVAL;
1734 for (i = 0; i < num; i++) {
1735 ret = register_kprobe(kps[i]);
1736 if (ret < 0) {
1737 if (i > 0)
1738 unregister_kprobes(kps, i);
1739 break;
1740 }
1741 }
1742 return ret;
1743}
1744EXPORT_SYMBOL_GPL(register_kprobes);
1745
1746void unregister_kprobe(struct kprobe *p)
1747{
1748 unregister_kprobes(&p, 1);
1749}
1750EXPORT_SYMBOL_GPL(unregister_kprobe);
1751
1752void unregister_kprobes(struct kprobe **kps, int num)
1753{
1754 int i;
1755
1756 if (num <= 0)
1757 return;
1758 mutex_lock(&kprobe_mutex);
1759 for (i = 0; i < num; i++)
1760 if (__unregister_kprobe_top(kps[i]) < 0)
1761 kps[i]->addr = NULL;
1762 mutex_unlock(&kprobe_mutex);
1763
1764 synchronize_sched();
1765 for (i = 0; i < num; i++)
1766 if (kps[i]->addr)
1767 __unregister_kprobe_bottom(kps[i]);
1768}
1769EXPORT_SYMBOL_GPL(unregister_kprobes);
1770
1771int __weak kprobe_exceptions_notify(struct notifier_block *self,
1772 unsigned long val, void *data)
1773{
1774 return NOTIFY_DONE;
1775}
1776NOKPROBE_SYMBOL(kprobe_exceptions_notify);
1777
1778static struct notifier_block kprobe_exceptions_nb = {
1779 .notifier_call = kprobe_exceptions_notify,
1780 .priority = 0x7fffffff /* we need to be notified first */
1781};
1782
1783unsigned long __weak arch_deref_entry_point(void *entry)
1784{
1785 return (unsigned long)entry;
1786}
1787
1788#ifdef CONFIG_KRETPROBES
1789/*
1790 * This kprobe pre_handler is registered with every kretprobe. When probe
1791 * hits it will set up the return probe.
1792 */
1793static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
1794{
1795 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1796 unsigned long hash, flags = 0;
1797 struct kretprobe_instance *ri;
1798
1799 /*
1800 * To avoid deadlocks, prohibit return probing in NMI contexts,
1801 * just skip the probe and increase the (inexact) 'nmissed'
1802 * statistical counter, so that the user is informed that
1803 * something happened:
1804 */
1805 if (unlikely(in_nmi())) {
1806 rp->nmissed++;
1807 return 0;
1808 }
1809
1810 /* TODO: consider to only swap the RA after the last pre_handler fired */
1811 hash = hash_ptr(current, KPROBE_HASH_BITS);
1812 raw_spin_lock_irqsave(&rp->lock, flags);
1813 if (!hlist_empty(&rp->free_instances)) {
1814 ri = hlist_entry(rp->free_instances.first,
1815 struct kretprobe_instance, hlist);
1816 hlist_del(&ri->hlist);
1817 raw_spin_unlock_irqrestore(&rp->lock, flags);
1818
1819 ri->rp = rp;
1820 ri->task = current;
1821
1822 if (rp->entry_handler && rp->entry_handler(ri, regs)) {
1823 raw_spin_lock_irqsave(&rp->lock, flags);
1824 hlist_add_head(&ri->hlist, &rp->free_instances);
1825 raw_spin_unlock_irqrestore(&rp->lock, flags);
1826 return 0;
1827 }
1828
1829 arch_prepare_kretprobe(ri, regs);
1830
1831 /* XXX(hch): why is there no hlist_move_head? */
1832 INIT_HLIST_NODE(&ri->hlist);
1833 kretprobe_table_lock(hash, &flags);
1834 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1835 kretprobe_table_unlock(hash, &flags);
1836 } else {
1837 rp->nmissed++;
1838 raw_spin_unlock_irqrestore(&rp->lock, flags);
1839 }
1840 return 0;
1841}
1842NOKPROBE_SYMBOL(pre_handler_kretprobe);
1843
1844bool __weak arch_kprobe_on_func_entry(unsigned long offset)
1845{
1846 return !offset;
1847}
1848
1849bool kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
1850{
1851 kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
1852
1853 if (IS_ERR(kp_addr))
1854 return false;
1855
1856 if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset) ||
1857 !arch_kprobe_on_func_entry(offset))
1858 return false;
1859
1860 return true;
1861}
1862
1863int register_kretprobe(struct kretprobe *rp)
1864{
1865 int ret = 0;
1866 struct kretprobe_instance *inst;
1867 int i;
1868 void *addr;
1869
1870 if (!kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset))
1871 return -EINVAL;
1872
1873 if (kretprobe_blacklist_size) {
1874 addr = kprobe_addr(&rp->kp);
1875 if (IS_ERR(addr))
1876 return PTR_ERR(addr);
1877
1878 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1879 if (kretprobe_blacklist[i].addr == addr)
1880 return -EINVAL;
1881 }
1882 }
1883
1884 rp->kp.pre_handler = pre_handler_kretprobe;
1885 rp->kp.post_handler = NULL;
1886 rp->kp.fault_handler = NULL;
1887
1888 /* Pre-allocate memory for max kretprobe instances */
1889 if (rp->maxactive <= 0) {
1890#ifdef CONFIG_PREEMPT
1891 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
1892#else
1893 rp->maxactive = num_possible_cpus();
1894#endif
1895 }
1896 raw_spin_lock_init(&rp->lock);
1897 INIT_HLIST_HEAD(&rp->free_instances);
1898 for (i = 0; i < rp->maxactive; i++) {
1899 inst = kmalloc(sizeof(struct kretprobe_instance) +
1900 rp->data_size, GFP_KERNEL);
1901 if (inst == NULL) {
1902 free_rp_inst(rp);
1903 return -ENOMEM;
1904 }
1905 INIT_HLIST_NODE(&inst->hlist);
1906 hlist_add_head(&inst->hlist, &rp->free_instances);
1907 }
1908
1909 rp->nmissed = 0;
1910 /* Establish function entry probe point */
1911 ret = register_kprobe(&rp->kp);
1912 if (ret != 0)
1913 free_rp_inst(rp);
1914 return ret;
1915}
1916EXPORT_SYMBOL_GPL(register_kretprobe);
1917
1918int register_kretprobes(struct kretprobe **rps, int num)
1919{
1920 int ret = 0, i;
1921
1922 if (num <= 0)
1923 return -EINVAL;
1924 for (i = 0; i < num; i++) {
1925 ret = register_kretprobe(rps[i]);
1926 if (ret < 0) {
1927 if (i > 0)
1928 unregister_kretprobes(rps, i);
1929 break;
1930 }
1931 }
1932 return ret;
1933}
1934EXPORT_SYMBOL_GPL(register_kretprobes);
1935
1936void unregister_kretprobe(struct kretprobe *rp)
1937{
1938 unregister_kretprobes(&rp, 1);
1939}
1940EXPORT_SYMBOL_GPL(unregister_kretprobe);
1941
1942void unregister_kretprobes(struct kretprobe **rps, int num)
1943{
1944 int i;
1945
1946 if (num <= 0)
1947 return;
1948 mutex_lock(&kprobe_mutex);
1949 for (i = 0; i < num; i++)
1950 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1951 rps[i]->kp.addr = NULL;
1952 mutex_unlock(&kprobe_mutex);
1953
1954 synchronize_sched();
1955 for (i = 0; i < num; i++) {
1956 if (rps[i]->kp.addr) {
1957 __unregister_kprobe_bottom(&rps[i]->kp);
1958 cleanup_rp_inst(rps[i]);
1959 }
1960 }
1961}
1962EXPORT_SYMBOL_GPL(unregister_kretprobes);
1963
1964#else /* CONFIG_KRETPROBES */
1965int register_kretprobe(struct kretprobe *rp)
1966{
1967 return -ENOSYS;
1968}
1969EXPORT_SYMBOL_GPL(register_kretprobe);
1970
1971int register_kretprobes(struct kretprobe **rps, int num)
1972{
1973 return -ENOSYS;
1974}
1975EXPORT_SYMBOL_GPL(register_kretprobes);
1976
1977void unregister_kretprobe(struct kretprobe *rp)
1978{
1979}
1980EXPORT_SYMBOL_GPL(unregister_kretprobe);
1981
1982void unregister_kretprobes(struct kretprobe **rps, int num)
1983{
1984}
1985EXPORT_SYMBOL_GPL(unregister_kretprobes);
1986
1987static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
1988{
1989 return 0;
1990}
1991NOKPROBE_SYMBOL(pre_handler_kretprobe);
1992
1993#endif /* CONFIG_KRETPROBES */
1994
1995/* Set the kprobe gone and remove its instruction buffer. */
1996static void kill_kprobe(struct kprobe *p)
1997{
1998 struct kprobe *kp;
1999
2000 p->flags |= KPROBE_FLAG_GONE;
2001 if (kprobe_aggrprobe(p)) {
2002 /*
2003 * If this is an aggr_kprobe, we have to list all the
2004 * chained probes and mark them GONE.
2005 */
2006 list_for_each_entry_rcu(kp, &p->list, list)
2007 kp->flags |= KPROBE_FLAG_GONE;
2008 p->post_handler = NULL;
2009 kill_optimized_kprobe(p);
2010 }
2011 /*
2012 * Here, we can remove insn_slot safely, because no thread calls
2013 * the original probed function (which will be freed soon) any more.
2014 */
2015 arch_remove_kprobe(p);
2016}
2017
2018/* Disable one kprobe */
2019int disable_kprobe(struct kprobe *kp)
2020{
2021 int ret = 0;
2022 struct kprobe *p;
2023
2024 mutex_lock(&kprobe_mutex);
2025
2026 /* Disable this kprobe */
2027 p = __disable_kprobe(kp);
2028 if (IS_ERR(p))
2029 ret = PTR_ERR(p);
2030
2031 mutex_unlock(&kprobe_mutex);
2032 return ret;
2033}
2034EXPORT_SYMBOL_GPL(disable_kprobe);
2035
2036/* Enable one kprobe */
2037int enable_kprobe(struct kprobe *kp)
2038{
2039 int ret = 0;
2040 struct kprobe *p;
2041
2042 mutex_lock(&kprobe_mutex);
2043
2044 /* Check whether specified probe is valid. */
2045 p = __get_valid_kprobe(kp);
2046 if (unlikely(p == NULL)) {
2047 ret = -EINVAL;
2048 goto out;
2049 }
2050
2051 if (kprobe_gone(kp)) {
2052 /* This kprobe has gone, we couldn't enable it. */
2053 ret = -EINVAL;
2054 goto out;
2055 }
2056
2057 if (p != kp)
2058 kp->flags &= ~KPROBE_FLAG_DISABLED;
2059
2060 if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2061 p->flags &= ~KPROBE_FLAG_DISABLED;
2062 ret = arm_kprobe(p);
2063 if (ret)
2064 p->flags |= KPROBE_FLAG_DISABLED;
2065 }
2066out:
2067 mutex_unlock(&kprobe_mutex);
2068 return ret;
2069}
2070EXPORT_SYMBOL_GPL(enable_kprobe);
2071
2072/* Caller must NOT call this in usual path. This is only for critical case */
2073void dump_kprobe(struct kprobe *kp)
2074{
2075 pr_err("Dumping kprobe:\n");
2076 pr_err("Name: %s\nOffset: %x\nAddress: %pS\n",
2077 kp->symbol_name, kp->offset, kp->addr);
2078}
2079NOKPROBE_SYMBOL(dump_kprobe);
2080
2081/*
2082 * Lookup and populate the kprobe_blacklist.
2083 *
2084 * Unlike the kretprobe blacklist, we'll need to determine
2085 * the range of addresses that belong to the said functions,
2086 * since a kprobe need not necessarily be at the beginning
2087 * of a function.
2088 */
2089static int __init populate_kprobe_blacklist(unsigned long *start,
2090 unsigned long *end)
2091{
2092 unsigned long *iter;
2093 struct kprobe_blacklist_entry *ent;
2094 unsigned long entry, offset = 0, size = 0;
2095
2096 for (iter = start; iter < end; iter++) {
2097 entry = arch_deref_entry_point((void *)*iter);
2098
2099 if (!kernel_text_address(entry) ||
2100 !kallsyms_lookup_size_offset(entry, &size, &offset))
2101 continue;
2102
2103 ent = kmalloc(sizeof(*ent), GFP_KERNEL);
2104 if (!ent)
2105 return -ENOMEM;
2106 ent->start_addr = entry;
2107 ent->end_addr = entry + size;
2108 INIT_LIST_HEAD(&ent->list);
2109 list_add_tail(&ent->list, &kprobe_blacklist);
2110 }
2111 return 0;
2112}
2113
2114/* Module notifier call back, checking kprobes on the module */
2115static int kprobes_module_callback(struct notifier_block *nb,
2116 unsigned long val, void *data)
2117{
2118 struct module *mod = data;
2119 struct hlist_head *head;
2120 struct kprobe *p;
2121 unsigned int i;
2122 int checkcore = (val == MODULE_STATE_GOING);
2123
2124 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
2125 return NOTIFY_DONE;
2126
2127 /*
2128 * When MODULE_STATE_GOING was notified, both of module .text and
2129 * .init.text sections would be freed. When MODULE_STATE_LIVE was
2130 * notified, only .init.text section would be freed. We need to
2131 * disable kprobes which have been inserted in the sections.
2132 */
2133 mutex_lock(&kprobe_mutex);
2134 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2135 head = &kprobe_table[i];
2136 hlist_for_each_entry_rcu(p, head, hlist)
2137 if (within_module_init((unsigned long)p->addr, mod) ||
2138 (checkcore &&
2139 within_module_core((unsigned long)p->addr, mod))) {
2140 /*
2141 * The vaddr this probe is installed will soon
2142 * be vfreed buy not synced to disk. Hence,
2143 * disarming the breakpoint isn't needed.
2144 *
2145 * Note, this will also move any optimized probes
2146 * that are pending to be removed from their
2147 * corresponding lists to the freeing_list and
2148 * will not be touched by the delayed
2149 * kprobe_optimizer work handler.
2150 */
2151 kill_kprobe(p);
2152 }
2153 }
2154 mutex_unlock(&kprobe_mutex);
2155 return NOTIFY_DONE;
2156}
2157
2158static struct notifier_block kprobe_module_nb = {
2159 .notifier_call = kprobes_module_callback,
2160 .priority = 0
2161};
2162
2163/* Markers of _kprobe_blacklist section */
2164extern unsigned long __start_kprobe_blacklist[];
2165extern unsigned long __stop_kprobe_blacklist[];
2166
2167static int __init init_kprobes(void)
2168{
2169 int i, err = 0;
2170
2171 /* FIXME allocate the probe table, currently defined statically */
2172 /* initialize all list heads */
2173 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2174 INIT_HLIST_HEAD(&kprobe_table[i]);
2175 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
2176 raw_spin_lock_init(&(kretprobe_table_locks[i].lock));
2177 }
2178
2179 err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2180 __stop_kprobe_blacklist);
2181 if (err) {
2182 pr_err("kprobes: failed to populate blacklist: %d\n", err);
2183 pr_err("Please take care of using kprobes.\n");
2184 }
2185
2186 if (kretprobe_blacklist_size) {
2187 /* lookup the function address from its name */
2188 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2189 kretprobe_blacklist[i].addr =
2190 kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
2191 if (!kretprobe_blacklist[i].addr)
2192 printk("kretprobe: lookup failed: %s\n",
2193 kretprobe_blacklist[i].name);
2194 }
2195 }
2196
2197#if defined(CONFIG_OPTPROBES)
2198#if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2199 /* Init kprobe_optinsn_slots */
2200 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2201#endif
2202 /* By default, kprobes can be optimized */
2203 kprobes_allow_optimization = true;
2204#endif
2205
2206 /* By default, kprobes are armed */
2207 kprobes_all_disarmed = false;
2208
2209 err = arch_init_kprobes();
2210 if (!err)
2211 err = register_die_notifier(&kprobe_exceptions_nb);
2212 if (!err)
2213 err = register_module_notifier(&kprobe_module_nb);
2214
2215 kprobes_initialized = (err == 0);
2216
2217 if (!err)
2218 init_test_probes();
2219 return err;
2220}
2221
2222#ifdef CONFIG_DEBUG_FS
2223static void report_probe(struct seq_file *pi, struct kprobe *p,
2224 const char *sym, int offset, char *modname, struct kprobe *pp)
2225{
2226 char *kprobe_type;
2227 void *addr = p->addr;
2228
2229 if (p->pre_handler == pre_handler_kretprobe)
2230 kprobe_type = "r";
2231 else
2232 kprobe_type = "k";
2233
2234 if (!kallsyms_show_value())
2235 addr = NULL;
2236
2237 if (sym)
2238 seq_printf(pi, "%px %s %s+0x%x %s ",
2239 addr, kprobe_type, sym, offset,
2240 (modname ? modname : " "));
2241 else /* try to use %pS */
2242 seq_printf(pi, "%px %s %pS ",
2243 addr, kprobe_type, p->addr);
2244
2245 if (!pp)
2246 pp = p;
2247 seq_printf(pi, "%s%s%s%s\n",
2248 (kprobe_gone(p) ? "[GONE]" : ""),
2249 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
2250 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2251 (kprobe_ftrace(pp) ? "[FTRACE]" : ""));
2252}
2253
2254static void *kprobe_seq_start(struct seq_file *f, loff_t *pos)
2255{
2256 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2257}
2258
2259static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
2260{
2261 (*pos)++;
2262 if (*pos >= KPROBE_TABLE_SIZE)
2263 return NULL;
2264 return pos;
2265}
2266
2267static void kprobe_seq_stop(struct seq_file *f, void *v)
2268{
2269 /* Nothing to do */
2270}
2271
2272static int show_kprobe_addr(struct seq_file *pi, void *v)
2273{
2274 struct hlist_head *head;
2275 struct kprobe *p, *kp;
2276 const char *sym = NULL;
2277 unsigned int i = *(loff_t *) v;
2278 unsigned long offset = 0;
2279 char *modname, namebuf[KSYM_NAME_LEN];
2280
2281 head = &kprobe_table[i];
2282 preempt_disable();
2283 hlist_for_each_entry_rcu(p, head, hlist) {
2284 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
2285 &offset, &modname, namebuf);
2286 if (kprobe_aggrprobe(p)) {
2287 list_for_each_entry_rcu(kp, &p->list, list)
2288 report_probe(pi, kp, sym, offset, modname, p);
2289 } else
2290 report_probe(pi, p, sym, offset, modname, NULL);
2291 }
2292 preempt_enable();
2293 return 0;
2294}
2295
2296static const struct seq_operations kprobes_seq_ops = {
2297 .start = kprobe_seq_start,
2298 .next = kprobe_seq_next,
2299 .stop = kprobe_seq_stop,
2300 .show = show_kprobe_addr
2301};
2302
2303static int kprobes_open(struct inode *inode, struct file *filp)
2304{
2305 return seq_open(filp, &kprobes_seq_ops);
2306}
2307
2308static const struct file_operations debugfs_kprobes_operations = {
2309 .open = kprobes_open,
2310 .read = seq_read,
2311 .llseek = seq_lseek,
2312 .release = seq_release,
2313};
2314
2315/* kprobes/blacklist -- shows which functions can not be probed */
2316static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
2317{
2318 return seq_list_start(&kprobe_blacklist, *pos);
2319}
2320
2321static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos)
2322{
2323 return seq_list_next(v, &kprobe_blacklist, pos);
2324}
2325
2326static int kprobe_blacklist_seq_show(struct seq_file *m, void *v)
2327{
2328 struct kprobe_blacklist_entry *ent =
2329 list_entry(v, struct kprobe_blacklist_entry, list);
2330
2331 /*
2332 * If /proc/kallsyms is not showing kernel address, we won't
2333 * show them here either.
2334 */
2335 if (!kallsyms_show_value())
2336 seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL,
2337 (void *)ent->start_addr);
2338 else
2339 seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr,
2340 (void *)ent->end_addr, (void *)ent->start_addr);
2341 return 0;
2342}
2343
2344static const struct seq_operations kprobe_blacklist_seq_ops = {
2345 .start = kprobe_blacklist_seq_start,
2346 .next = kprobe_blacklist_seq_next,
2347 .stop = kprobe_seq_stop, /* Reuse void function */
2348 .show = kprobe_blacklist_seq_show,
2349};
2350
2351static int kprobe_blacklist_open(struct inode *inode, struct file *filp)
2352{
2353 return seq_open(filp, &kprobe_blacklist_seq_ops);
2354}
2355
2356static const struct file_operations debugfs_kprobe_blacklist_ops = {
2357 .open = kprobe_blacklist_open,
2358 .read = seq_read,
2359 .llseek = seq_lseek,
2360 .release = seq_release,
2361};
2362
2363static int arm_all_kprobes(void)
2364{
2365 struct hlist_head *head;
2366 struct kprobe *p;
2367 unsigned int i, total = 0, errors = 0;
2368 int err, ret = 0;
2369
2370 mutex_lock(&kprobe_mutex);
2371
2372 /* If kprobes are armed, just return */
2373 if (!kprobes_all_disarmed)
2374 goto already_enabled;
2375
2376 /*
2377 * optimize_kprobe() called by arm_kprobe() checks
2378 * kprobes_all_disarmed, so set kprobes_all_disarmed before
2379 * arm_kprobe.
2380 */
2381 kprobes_all_disarmed = false;
2382 /* Arming kprobes doesn't optimize kprobe itself */
2383 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2384 head = &kprobe_table[i];
2385 /* Arm all kprobes on a best-effort basis */
2386 hlist_for_each_entry_rcu(p, head, hlist) {
2387 if (!kprobe_disabled(p)) {
2388 err = arm_kprobe(p);
2389 if (err) {
2390 errors++;
2391 ret = err;
2392 }
2393 total++;
2394 }
2395 }
2396 }
2397
2398 if (errors)
2399 pr_warn("Kprobes globally enabled, but failed to arm %d out of %d probes\n",
2400 errors, total);
2401 else
2402 pr_info("Kprobes globally enabled\n");
2403
2404already_enabled:
2405 mutex_unlock(&kprobe_mutex);
2406 return ret;
2407}
2408
2409static int disarm_all_kprobes(void)
2410{
2411 struct hlist_head *head;
2412 struct kprobe *p;
2413 unsigned int i, total = 0, errors = 0;
2414 int err, ret = 0;
2415
2416 mutex_lock(&kprobe_mutex);
2417
2418 /* If kprobes are already disarmed, just return */
2419 if (kprobes_all_disarmed) {
2420 mutex_unlock(&kprobe_mutex);
2421 return 0;
2422 }
2423
2424 kprobes_all_disarmed = true;
2425
2426 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2427 head = &kprobe_table[i];
2428 /* Disarm all kprobes on a best-effort basis */
2429 hlist_for_each_entry_rcu(p, head, hlist) {
2430 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
2431 err = disarm_kprobe(p, false);
2432 if (err) {
2433 errors++;
2434 ret = err;
2435 }
2436 total++;
2437 }
2438 }
2439 }
2440
2441 if (errors)
2442 pr_warn("Kprobes globally disabled, but failed to disarm %d out of %d probes\n",
2443 errors, total);
2444 else
2445 pr_info("Kprobes globally disabled\n");
2446
2447 mutex_unlock(&kprobe_mutex);
2448
2449 /* Wait for disarming all kprobes by optimizer */
2450 wait_for_kprobe_optimizer();
2451
2452 return ret;
2453}
2454
2455/*
2456 * XXX: The debugfs bool file interface doesn't allow for callbacks
2457 * when the bool state is switched. We can reuse that facility when
2458 * available
2459 */
2460static ssize_t read_enabled_file_bool(struct file *file,
2461 char __user *user_buf, size_t count, loff_t *ppos)
2462{
2463 char buf[3];
2464
2465 if (!kprobes_all_disarmed)
2466 buf[0] = '1';
2467 else
2468 buf[0] = '0';
2469 buf[1] = '\n';
2470 buf[2] = 0x00;
2471 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2472}
2473
2474static ssize_t write_enabled_file_bool(struct file *file,
2475 const char __user *user_buf, size_t count, loff_t *ppos)
2476{
2477 char buf[32];
2478 size_t buf_size;
2479 int ret = 0;
2480
2481 buf_size = min(count, (sizeof(buf)-1));
2482 if (copy_from_user(buf, user_buf, buf_size))
2483 return -EFAULT;
2484
2485 buf[buf_size] = '\0';
2486 switch (buf[0]) {
2487 case 'y':
2488 case 'Y':
2489 case '1':
2490 ret = arm_all_kprobes();
2491 break;
2492 case 'n':
2493 case 'N':
2494 case '0':
2495 ret = disarm_all_kprobes();
2496 break;
2497 default:
2498 return -EINVAL;
2499 }
2500
2501 if (ret)
2502 return ret;
2503
2504 return count;
2505}
2506
2507static const struct file_operations fops_kp = {
2508 .read = read_enabled_file_bool,
2509 .write = write_enabled_file_bool,
2510 .llseek = default_llseek,
2511};
2512
2513static int __init debugfs_kprobe_init(void)
2514{
2515 struct dentry *dir, *file;
2516 unsigned int value = 1;
2517
2518 dir = debugfs_create_dir("kprobes", NULL);
2519 if (!dir)
2520 return -ENOMEM;
2521
2522 file = debugfs_create_file("list", 0400, dir, NULL,
2523 &debugfs_kprobes_operations);
2524 if (!file)
2525 goto error;
2526
2527 file = debugfs_create_file("enabled", 0600, dir,
2528 &value, &fops_kp);
2529 if (!file)
2530 goto error;
2531
2532 file = debugfs_create_file("blacklist", 0400, dir, NULL,
2533 &debugfs_kprobe_blacklist_ops);
2534 if (!file)
2535 goto error;
2536
2537 return 0;
2538
2539error:
2540 debugfs_remove(dir);
2541 return -ENOMEM;
2542}
2543
2544late_initcall(debugfs_kprobe_init);
2545#endif /* CONFIG_DEBUG_FS */
2546
2547module_init(init_kprobes);
2548