1 | #include <linux/string.h> |
2 | #include <linux/module.h> |
3 | #include <linux/io.h> |
4 | #include <linux/kmsan-checks.h> |
5 | |
6 | #define movs(type,to,from) \ |
7 | asm volatile("movs" type:"=&D" (to), "=&S" (from):"0" (to), "1" (from):"memory") |
8 | |
9 | /* Originally from i386/string.h */ |
10 | static __always_inline void rep_movs(void *to, const void *from, size_t n) |
11 | { |
12 | unsigned long d0, d1, d2; |
13 | asm volatile("rep ; movsl\n\t" |
14 | "testb $2,%b4\n\t" |
15 | "je 1f\n\t" |
16 | "movsw\n" |
17 | "1:\ttestb $1,%b4\n\t" |
18 | "je 2f\n\t" |
19 | "movsb\n" |
20 | "2:" |
21 | : "=&c" (d0), "=&D" (d1), "=&S" (d2) |
22 | : "0" (n / 4), "q" (n), "1" ((long)to), "2" ((long)from) |
23 | : "memory" ); |
24 | } |
25 | |
26 | static void string_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) |
27 | { |
28 | if (unlikely(!n)) |
29 | return; |
30 | |
31 | /* Align any unaligned source IO */ |
32 | if (unlikely(1 & (unsigned long)from)) { |
33 | movs("b" , to, from); |
34 | n--; |
35 | } |
36 | if (n > 1 && unlikely(2 & (unsigned long)from)) { |
37 | movs("w" , to, from); |
38 | n-=2; |
39 | } |
40 | rep_movs(to, from: (const void *)from, n); |
41 | /* KMSAN must treat values read from devices as initialized. */ |
42 | kmsan_unpoison_memory(address: to, size: n); |
43 | } |
44 | |
45 | static void string_memcpy_toio(volatile void __iomem *to, const void *from, size_t n) |
46 | { |
47 | if (unlikely(!n)) |
48 | return; |
49 | |
50 | /* Make sure uninitialized memory isn't copied to devices. */ |
51 | kmsan_check_memory(address: from, size: n); |
52 | /* Align any unaligned destination IO */ |
53 | if (unlikely(1 & (unsigned long)to)) { |
54 | movs("b" , to, from); |
55 | n--; |
56 | } |
57 | if (n > 1 && unlikely(2 & (unsigned long)to)) { |
58 | movs("w" , to, from); |
59 | n-=2; |
60 | } |
61 | rep_movs(to: (void *)to, from: (const void *) from, n); |
62 | } |
63 | |
64 | static void unrolled_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) |
65 | { |
66 | const volatile char __iomem *in = from; |
67 | char *out = to; |
68 | int i; |
69 | |
70 | for (i = 0; i < n; ++i) |
71 | out[i] = readb(addr: &in[i]); |
72 | } |
73 | |
74 | static void unrolled_memcpy_toio(volatile void __iomem *to, const void *from, size_t n) |
75 | { |
76 | volatile char __iomem *out = to; |
77 | const char *in = from; |
78 | int i; |
79 | |
80 | for (i = 0; i < n; ++i) |
81 | writeb(val: in[i], addr: &out[i]); |
82 | } |
83 | |
84 | static void unrolled_memset_io(volatile void __iomem *a, int b, size_t c) |
85 | { |
86 | volatile char __iomem *mem = a; |
87 | int i; |
88 | |
89 | for (i = 0; i < c; ++i) |
90 | writeb(val: b, addr: &mem[i]); |
91 | } |
92 | |
93 | void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) |
94 | { |
95 | if (cc_platform_has(attr: CC_ATTR_GUEST_UNROLL_STRING_IO)) |
96 | unrolled_memcpy_fromio(to, from, n); |
97 | else |
98 | string_memcpy_fromio(to, from, n); |
99 | } |
100 | EXPORT_SYMBOL(memcpy_fromio); |
101 | |
102 | void memcpy_toio(volatile void __iomem *to, const void *from, size_t n) |
103 | { |
104 | if (cc_platform_has(attr: CC_ATTR_GUEST_UNROLL_STRING_IO)) |
105 | unrolled_memcpy_toio(to, from, n); |
106 | else |
107 | string_memcpy_toio(to, from, n); |
108 | } |
109 | EXPORT_SYMBOL(memcpy_toio); |
110 | |
111 | void memset_io(volatile void __iomem *a, int b, size_t c) |
112 | { |
113 | if (cc_platform_has(attr: CC_ATTR_GUEST_UNROLL_STRING_IO)) { |
114 | unrolled_memset_io(a, b, c); |
115 | } else { |
116 | /* |
117 | * TODO: memset can mangle the IO patterns quite a bit. |
118 | * perhaps it would be better to use a dumb one: |
119 | */ |
120 | memset((void *)a, b, c); |
121 | } |
122 | } |
123 | EXPORT_SYMBOL(memset_io); |
124 | |