1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> |
4 | * Copyright 2002 Andi Kleen, SuSE Labs. |
5 | * |
6 | * Functions to copy from and to user space. |
7 | */ |
8 | |
9 | #include <linux/export.h> |
10 | #include <linux/linkage.h> |
11 | #include <asm/cpufeatures.h> |
12 | #include <asm/alternative.h> |
13 | #include <asm/asm.h> |
14 | |
15 | /* |
16 | * rep_movs_alternative - memory copy with exception handling. |
17 | * This version is for CPUs that don't have FSRM (Fast Short Rep Movs) |
18 | * |
19 | * Input: |
20 | * rdi destination |
21 | * rsi source |
22 | * rcx count |
23 | * |
24 | * Output: |
25 | * rcx uncopied bytes or 0 if successful. |
26 | * |
27 | * NOTE! The calling convention is very intentionally the same as |
28 | * for 'rep movs', so that we can rewrite the function call with |
29 | * just a plain 'rep movs' on machines that have FSRM. But to make |
30 | * it simpler for us, we can clobber rsi/rdi and rax freely. |
31 | */ |
32 | SYM_FUNC_START(rep_movs_alternative) |
33 | cmpq $64,%rcx |
34 | jae .Llarge |
35 | |
36 | cmp $8,%ecx |
37 | jae .Lword |
38 | |
39 | testl %ecx,%ecx |
40 | je .Lexit |
41 | |
42 | .Lcopy_user_tail: |
43 | 0: movb (%rsi),%al |
44 | 1: movb %al,(%rdi) |
45 | inc %rdi |
46 | inc %rsi |
47 | dec %rcx |
48 | jne .Lcopy_user_tail |
49 | .Lexit: |
50 | RET |
51 | |
52 | _ASM_EXTABLE_UA( 0b, .Lexit) |
53 | _ASM_EXTABLE_UA( 1b, .Lexit) |
54 | |
55 | .p2align 4 |
56 | .Lword: |
57 | 2: movq (%rsi),%rax |
58 | 3: movq %rax,(%rdi) |
59 | addq $8,%rsi |
60 | addq $8,%rdi |
61 | sub $8,%ecx |
62 | je .Lexit |
63 | cmp $8,%ecx |
64 | jae .Lword |
65 | jmp .Lcopy_user_tail |
66 | |
67 | _ASM_EXTABLE_UA( 2b, .Lcopy_user_tail) |
68 | _ASM_EXTABLE_UA( 3b, .Lcopy_user_tail) |
69 | |
70 | .Llarge: |
71 | 0: ALTERNATIVE "jmp .Llarge_movsq" , "rep movsb" , X86_FEATURE_ERMS |
72 | 1: RET |
73 | |
74 | _ASM_EXTABLE_UA( 0b, 1b) |
75 | |
76 | .Llarge_movsq: |
77 | movq %rcx,%rax |
78 | shrq $3,%rcx |
79 | andl $7,%eax |
80 | 0: rep movsq |
81 | movl %eax,%ecx |
82 | testl %ecx,%ecx |
83 | jne .Lcopy_user_tail |
84 | RET |
85 | |
86 | 1: leaq (%rax,%rcx,8),%rcx |
87 | jmp .Lcopy_user_tail |
88 | |
89 | _ASM_EXTABLE_UA( 0b, 1b) |
90 | SYM_FUNC_END(rep_movs_alternative) |
91 | EXPORT_SYMBOL(rep_movs_alternative) |
92 | |