1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2012 Intel Corporation |
4 | * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> |
5 | */ |
6 | |
7 | #include <linux/raid/pq.h> |
8 | |
9 | #ifdef __KERNEL__ |
10 | #include <asm/neon.h> |
11 | #include "neon.h" |
12 | #else |
13 | #define kernel_neon_begin() |
14 | #define kernel_neon_end() |
15 | #define cpu_has_neon() (1) |
16 | #endif |
17 | |
18 | static int raid6_has_neon(void) |
19 | { |
20 | return cpu_has_neon(); |
21 | } |
22 | |
23 | static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, |
24 | int failb, void **ptrs) |
25 | { |
26 | u8 *p, *q, *dp, *dq; |
27 | const u8 *pbmul; /* P multiplier table for B data */ |
28 | const u8 *qmul; /* Q multiplier table (for both) */ |
29 | |
30 | p = (u8 *)ptrs[disks - 2]; |
31 | q = (u8 *)ptrs[disks - 1]; |
32 | |
33 | /* |
34 | * Compute syndrome with zero for the missing data pages |
35 | * Use the dead data pages as temporary storage for |
36 | * delta p and delta q |
37 | */ |
38 | dp = (u8 *)ptrs[faila]; |
39 | ptrs[faila] = (void *)raid6_empty_zero_page; |
40 | ptrs[disks - 2] = dp; |
41 | dq = (u8 *)ptrs[failb]; |
42 | ptrs[failb] = (void *)raid6_empty_zero_page; |
43 | ptrs[disks - 1] = dq; |
44 | |
45 | raid6_call.gen_syndrome(disks, bytes, ptrs); |
46 | |
47 | /* Restore pointer table */ |
48 | ptrs[faila] = dp; |
49 | ptrs[failb] = dq; |
50 | ptrs[disks - 2] = p; |
51 | ptrs[disks - 1] = q; |
52 | |
53 | /* Now, pick the proper data tables */ |
54 | pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]]; |
55 | qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ |
56 | raid6_gfexp[failb]]]; |
57 | |
58 | kernel_neon_begin(); |
59 | __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul); |
60 | kernel_neon_end(); |
61 | } |
62 | |
63 | static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, |
64 | void **ptrs) |
65 | { |
66 | u8 *p, *q, *dq; |
67 | const u8 *qmul; /* Q multiplier table */ |
68 | |
69 | p = (u8 *)ptrs[disks - 2]; |
70 | q = (u8 *)ptrs[disks - 1]; |
71 | |
72 | /* |
73 | * Compute syndrome with zero for the missing data page |
74 | * Use the dead data page as temporary storage for delta q |
75 | */ |
76 | dq = (u8 *)ptrs[faila]; |
77 | ptrs[faila] = (void *)raid6_empty_zero_page; |
78 | ptrs[disks - 1] = dq; |
79 | |
80 | raid6_call.gen_syndrome(disks, bytes, ptrs); |
81 | |
82 | /* Restore pointer table */ |
83 | ptrs[faila] = dq; |
84 | ptrs[disks - 1] = q; |
85 | |
86 | /* Now, pick the proper data tables */ |
87 | qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; |
88 | |
89 | kernel_neon_begin(); |
90 | __raid6_datap_recov_neon(bytes, p, q, dq, qmul); |
91 | kernel_neon_end(); |
92 | } |
93 | |
94 | const struct raid6_recov_calls raid6_recov_neon = { |
95 | .data2 = raid6_2data_recov_neon, |
96 | .datap = raid6_datap_recov_neon, |
97 | .valid = raid6_has_neon, |
98 | .name = "neon" , |
99 | .priority = 10, |
100 | }; |
101 | |