1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */ |
3 | |
4 | #include <linux/jump_label.h> |
5 | #include <linux/uaccess.h> |
6 | #include <linux/export.h> |
7 | #include <linux/string.h> |
8 | #include <linux/types.h> |
9 | |
10 | #include <asm/mce.h> |
11 | |
12 | #ifdef CONFIG_X86_MCE |
13 | static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key); |
14 | |
15 | void enable_copy_mc_fragile(void) |
16 | { |
17 | static_branch_inc(©_mc_fragile_key); |
18 | } |
19 | #define copy_mc_fragile_enabled (static_branch_unlikely(©_mc_fragile_key)) |
20 | |
21 | /* |
22 | * Similar to copy_user_handle_tail, probe for the write fault point, or |
23 | * source exception point. |
24 | */ |
25 | __visible notrace unsigned long |
26 | copy_mc_fragile_handle_tail(char *to, char *from, unsigned len) |
27 | { |
28 | for (; len; --len, to++, from++) |
29 | if (copy_mc_fragile(dst: to, src: from, cnt: 1)) |
30 | break; |
31 | return len; |
32 | } |
33 | #else |
34 | /* |
35 | * No point in doing careful copying, or consulting a static key when |
36 | * there is no #MC handler in the CONFIG_X86_MCE=n case. |
37 | */ |
38 | void enable_copy_mc_fragile(void) |
39 | { |
40 | } |
41 | #define copy_mc_fragile_enabled (0) |
42 | #endif |
43 | |
44 | unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len); |
45 | |
46 | /** |
47 | * copy_mc_to_kernel - memory copy that handles source exceptions |
48 | * |
49 | * @dst: destination address |
50 | * @src: source address |
51 | * @len: number of bytes to copy |
52 | * |
53 | * Call into the 'fragile' version on systems that benefit from avoiding |
54 | * corner case poison consumption scenarios, For example, accessing |
55 | * poison across 2 cachelines with a single instruction. Almost all |
56 | * other uses case can use copy_mc_enhanced_fast_string() for a fast |
57 | * recoverable copy, or fallback to plain memcpy. |
58 | * |
59 | * Return 0 for success, or number of bytes not copied if there was an |
60 | * exception. |
61 | */ |
62 | unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len) |
63 | { |
64 | if (copy_mc_fragile_enabled) |
65 | return copy_mc_fragile(dst, src, cnt: len); |
66 | if (static_cpu_has(X86_FEATURE_ERMS)) |
67 | return copy_mc_enhanced_fast_string(dst, src, len); |
68 | memcpy(dst, src, len); |
69 | return 0; |
70 | } |
71 | EXPORT_SYMBOL_GPL(copy_mc_to_kernel); |
72 | |
73 | unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, unsigned len) |
74 | { |
75 | unsigned long ret; |
76 | |
77 | if (copy_mc_fragile_enabled) { |
78 | __uaccess_begin(); |
79 | ret = copy_mc_fragile(dst: (__force void *)dst, src, cnt: len); |
80 | __uaccess_end(); |
81 | return ret; |
82 | } |
83 | |
84 | if (static_cpu_has(X86_FEATURE_ERMS)) { |
85 | __uaccess_begin(); |
86 | ret = copy_mc_enhanced_fast_string(dst: (__force void *)dst, src, len); |
87 | __uaccess_end(); |
88 | return ret; |
89 | } |
90 | |
91 | return copy_user_generic(to: (__force void *)dst, from: src, len); |
92 | } |
93 | |