1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* |
3 | * |
4 | * Copyright (C) IBM Corporation, 2012 |
5 | * |
6 | * Author: Anton Blanchard <anton@au.ibm.com> |
7 | */ |
8 | #include <asm/page.h> |
9 | #include <asm/ppc_asm.h> |
10 | |
11 | _GLOBAL(copypage_power7) |
12 | /* |
13 | * We prefetch both the source and destination using enhanced touch |
14 | * instructions. We use a stream ID of 0 for the load side and |
15 | * 1 for the store side. Since source and destination are page |
16 | * aligned we don't need to clear the bottom 7 bits of either |
17 | * address. |
18 | */ |
19 | ori r9,r3,1 /* stream=1 => to */ |
20 | |
21 | #ifdef CONFIG_PPC_64K_PAGES |
22 | lis r7,0x0E01 /* depth=7 |
23 | * units/cachelines=512 */ |
24 | #else |
25 | lis r7,0x0E00 /* depth=7 */ |
26 | ori r7,r7,0x1000 /* units/cachelines=32 */ |
27 | #endif |
28 | ori r10,r7,1 /* stream=1 */ |
29 | |
30 | DCBT_SETUP_STREAMS(r4, r7, r9, r10, r8) |
31 | |
32 | #ifdef CONFIG_ALTIVEC |
33 | mflr r0 |
34 | std r3,-STACKFRAMESIZE+STK_REG(R31)(r1) |
35 | std r4,-STACKFRAMESIZE+STK_REG(R30)(r1) |
36 | std r0,16(r1) |
37 | stdu r1,-STACKFRAMESIZE(r1) |
38 | bl CFUNC(enter_vmx_ops) |
39 | cmpwi r3,0 |
40 | ld r0,STACKFRAMESIZE+16(r1) |
41 | ld r3,STK_REG(R31)(r1) |
42 | ld r4,STK_REG(R30)(r1) |
43 | mtlr r0 |
44 | |
45 | li r0,(PAGE_SIZE/128) |
46 | mtctr r0 |
47 | |
48 | beq .Lnonvmx_copy |
49 | |
50 | addi r1,r1,STACKFRAMESIZE |
51 | |
52 | li r6,16 |
53 | li r7,32 |
54 | li r8,48 |
55 | li r9,64 |
56 | li r10,80 |
57 | li r11,96 |
58 | li r12,112 |
59 | |
60 | .align 5 |
61 | 1: lvx v7,0,r4 |
62 | lvx v6,r4,r6 |
63 | lvx v5,r4,r7 |
64 | lvx v4,r4,r8 |
65 | lvx v3,r4,r9 |
66 | lvx v2,r4,r10 |
67 | lvx v1,r4,r11 |
68 | lvx v0,r4,r12 |
69 | addi r4,r4,128 |
70 | stvx v7,0,r3 |
71 | stvx v6,r3,r6 |
72 | stvx v5,r3,r7 |
73 | stvx v4,r3,r8 |
74 | stvx v3,r3,r9 |
75 | stvx v2,r3,r10 |
76 | stvx v1,r3,r11 |
77 | stvx v0,r3,r12 |
78 | addi r3,r3,128 |
79 | bdnz 1b |
80 | |
81 | b CFUNC(exit_vmx_ops) /* tail call optimise */ |
82 | |
83 | #else |
84 | li r0,(PAGE_SIZE/128) |
85 | mtctr r0 |
86 | |
87 | stdu r1,-STACKFRAMESIZE(r1) |
88 | #endif |
89 | |
90 | .Lnonvmx_copy: |
91 | std r14,STK_REG(R14)(r1) |
92 | std r15,STK_REG(R15)(r1) |
93 | std r16,STK_REG(R16)(r1) |
94 | std r17,STK_REG(R17)(r1) |
95 | std r18,STK_REG(R18)(r1) |
96 | std r19,STK_REG(R19)(r1) |
97 | std r20,STK_REG(R20)(r1) |
98 | |
99 | 1: ld r0,0(r4) |
100 | ld r5,8(r4) |
101 | ld r6,16(r4) |
102 | ld r7,24(r4) |
103 | ld r8,32(r4) |
104 | ld r9,40(r4) |
105 | ld r10,48(r4) |
106 | ld r11,56(r4) |
107 | ld r12,64(r4) |
108 | ld r14,72(r4) |
109 | ld r15,80(r4) |
110 | ld r16,88(r4) |
111 | ld r17,96(r4) |
112 | ld r18,104(r4) |
113 | ld r19,112(r4) |
114 | ld r20,120(r4) |
115 | addi r4,r4,128 |
116 | std r0,0(r3) |
117 | std r5,8(r3) |
118 | std r6,16(r3) |
119 | std r7,24(r3) |
120 | std r8,32(r3) |
121 | std r9,40(r3) |
122 | std r10,48(r3) |
123 | std r11,56(r3) |
124 | std r12,64(r3) |
125 | std r14,72(r3) |
126 | std r15,80(r3) |
127 | std r16,88(r3) |
128 | std r17,96(r3) |
129 | std r18,104(r3) |
130 | std r19,112(r3) |
131 | std r20,120(r3) |
132 | addi r3,r3,128 |
133 | bdnz 1b |
134 | |
135 | ld r14,STK_REG(R14)(r1) |
136 | ld r15,STK_REG(R15)(r1) |
137 | ld r16,STK_REG(R16)(r1) |
138 | ld r17,STK_REG(R17)(r1) |
139 | ld r18,STK_REG(R18)(r1) |
140 | ld r19,STK_REG(R19)(r1) |
141 | ld r20,STK_REG(R20)(r1) |
142 | addi r1,r1,STACKFRAMESIZE |
143 | blr |
144 | |