1 | /* Shared HTM header. Work around false transactional execution facility |
2 | intrinsics. |
3 | |
4 | Copyright (C) 2016-2022 Free Software Foundation, Inc. |
5 | This file is part of the GNU C Library. |
6 | |
7 | The GNU C Library is free software; you can redistribute it and/or |
8 | modify it under the terms of the GNU Lesser General Public |
9 | License as published by the Free Software Foundation; either |
10 | version 2.1 of the License, or (at your option) any later version. |
11 | |
12 | The GNU C Library is distributed in the hope that it will be useful, |
13 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
15 | Lesser General Public License for more details. |
16 | |
17 | You should have received a copy of the GNU Lesser General Public |
18 | License along with the GNU C Library; if not, see |
19 | <https://www.gnu.org/licenses/>. */ |
20 | |
21 | #ifndef _HTM_H |
22 | #define _HTM_H 1 |
23 | |
24 | #include <htmintrin.h> |
25 | |
26 | #ifdef __s390x__ |
27 | # define TX_FPRS_BYTES 64 |
28 | # define TX_SAVE_FPRS \ |
29 | " std %%f8, 0(%[R_FPRS])\n\t" \ |
30 | " std %%f9, 8(%[R_FPRS])\n\t" \ |
31 | " std %%f10, 16(%[R_FPRS])\n\t" \ |
32 | " std %%f11, 24(%[R_FPRS])\n\t" \ |
33 | " std %%f12, 32(%[R_FPRS])\n\t" \ |
34 | " std %%f13, 40(%[R_FPRS])\n\t" \ |
35 | " std %%f14, 48(%[R_FPRS])\n\t" \ |
36 | " std %%f15, 56(%[R_FPRS])\n\t" |
37 | |
38 | # define TX_RESTORE_FPRS \ |
39 | " ld %%f8, 0(%[R_FPRS])\n\t" \ |
40 | " ld %%f9, 8(%[R_FPRS])\n\t" \ |
41 | " ld %%f10, 16(%[R_FPRS])\n\t" \ |
42 | " ld %%f11, 24(%[R_FPRS])\n\t" \ |
43 | " ld %%f12, 32(%[R_FPRS])\n\t" \ |
44 | " ld %%f13, 40(%[R_FPRS])\n\t" \ |
45 | " ld %%f14, 48(%[R_FPRS])\n\t" \ |
46 | " ld %%f15, 56(%[R_FPRS])\n\t" |
47 | |
48 | #else |
49 | |
50 | # define TX_FPRS_BYTES 16 |
51 | # define TX_SAVE_FPRS \ |
52 | " std %%f4, 0(%[R_FPRS])\n\t" \ |
53 | " std %%f6, 8(%[R_FPRS])\n\t" |
54 | |
55 | # define TX_RESTORE_FPRS \ |
56 | " ld %%f4, 0(%[R_FPRS])\n\t" \ |
57 | " ld %%f6, 8(%[R_FPRS])\n\t" |
58 | |
59 | #endif /* ! __s390x__ */ |
60 | |
61 | /* Use own inline assembly instead of __builtin_tbegin, as tbegin |
62 | has to filter program interruptions which can't be done with the builtin. |
63 | Now the fprs have to be saved / restored here, too. |
64 | The fpc is also not saved / restored with the builtin. |
65 | The used inline assembly does not clobber the volatile fprs / vrs! |
66 | Clobbering the latter ones would force the compiler to save / restore |
67 | the call saved fprs as those overlap with the vrs, but they only need to be |
68 | restored if the transaction fails but not if the transaction is successfully |
69 | started. Thus the user of the tbegin macros in this header file has to |
70 | compile the file / function with -msoft-float. It prevents gcc from using |
71 | fprs / vrs. */ |
72 | #define __libc_tbegin(tdb) __libc_tbegin_base(tdb,,,) |
73 | |
74 | #define __libc_tbegin_retry_output_regs , [R_TX_CNT] "+&d" (__tx_cnt) |
75 | #define __libc_tbegin_retry_input_regs(retry_cnt) , [R_RETRY] "d" (retry_cnt) |
76 | #define __libc_tbegin_retry_abort_path_insn \ |
77 | /* If tbegin returned _HTM_TBEGIN_TRANSIENT, retry immediately so \ |
78 | that max tbegin_cnt transactions are tried. Otherwise return and \ |
79 | let the caller of this macro do the fallback path. */ \ |
80 | " jnh 1f\n\t" /* cc 1/3: jump to fallback path. */ \ |
81 | /* tbegin returned _HTM_TBEGIN_TRANSIENT: retry with transaction. */ \ |
82 | " crje %[R_TX_CNT], %[R_RETRY], 1f\n\t" /* Reached max retries? */ \ |
83 | " ahi %[R_TX_CNT], 1\n\t" \ |
84 | " ppa %[R_TX_CNT], 0, 1\n\t" /* Transaction-Abort Assist. */ \ |
85 | " j 2b\n\t" /* Loop to tbegin. */ |
86 | |
87 | /* Same as __libc_tbegin except if tbegin aborts with _HTM_TBEGIN_TRANSIENT. |
88 | Then this macros restores the fpc, fprs and automatically retries up to |
89 | retry_cnt tbegins. Further saving of the state is omitted as it is already |
90 | saved. This macro calls tbegin at most as retry_cnt + 1 times. */ |
91 | #define __libc_tbegin_retry(tdb, retry_cnt) \ |
92 | ({ int __ret; \ |
93 | int __tx_cnt = 0; \ |
94 | __ret = __libc_tbegin_base(tdb, \ |
95 | __libc_tbegin_retry_abort_path_insn, \ |
96 | __libc_tbegin_retry_output_regs, \ |
97 | __libc_tbegin_retry_input_regs(retry_cnt)); \ |
98 | __ret; \ |
99 | }) |
100 | |
101 | #define __libc_tbegin_base(tdb, abort_path_insn, output_regs, input_regs) \ |
102 | ({ int __ret; \ |
103 | int __fpc; \ |
104 | char __fprs[TX_FPRS_BYTES]; \ |
105 | __asm__ __volatile__ (".machine push\n\t" \ |
106 | ".machinemode \"zarch_nohighgprs\"\n\t" \ |
107 | ".machine \"all\"\n\t" \ |
108 | /* Save state at the outermost transaction. \ |
109 | As extracting nesting depth is expensive \ |
110 | on at least zEC12, save fprs at inner \ |
111 | transactions, too. \ |
112 | The fpc and fprs are saved here as they \ |
113 | are not saved by tbegin. There exist no \ |
114 | call-saved vrs, thus they are not saved \ |
115 | here. */ \ |
116 | " efpc %[R_FPC]\n\t" \ |
117 | TX_SAVE_FPRS \ |
118 | /* Begin transaction: save all gprs, allow \ |
119 | ar modification and fp operations. Some \ |
120 | program-interruptions (e.g. a null \ |
121 | pointer access) are filtered and the \ |
122 | transaction will abort. In this case \ |
123 | the normal lock path will execute it \ |
124 | again and result in a core dump wich does \ |
125 | now show at tbegin but the real executed \ |
126 | instruction. \ |
127 | However it is not guaranteed that this \ |
128 | retry operate on the same data and thus \ |
129 | may not end in an program-interruption. \ |
130 | Note: This could also be used to probe \ |
131 | memory for being accessible! */ \ |
132 | "2: tbegin 0, 0xFF0E\n\t" \ |
133 | /* Branch away in abort case (this is the \ |
134 | prefered sequence. See PoP in chapter 5 \ |
135 | Transactional-Execution Facility \ |
136 | Operation). */ \ |
137 | " jnz 0f\n\t" \ |
138 | /* Transaction has successfully started. */ \ |
139 | " lhi %[R_RET], 0\n\t" \ |
140 | " j 1f\n\t" \ |
141 | /* Transaction has aborted. Now we are at \ |
142 | the outermost transaction. Restore fprs \ |
143 | and fpc. */ \ |
144 | "0: ipm %[R_RET]\n\t" \ |
145 | " srl %[R_RET], 28\n\t" \ |
146 | " sfpc %[R_FPC]\n\t" \ |
147 | TX_RESTORE_FPRS \ |
148 | abort_path_insn \ |
149 | "1:\n\t" \ |
150 | ".machine pop\n" \ |
151 | : [R_RET] "=&d" (__ret), \ |
152 | [R_FPC] "=&d" (__fpc) \ |
153 | output_regs \ |
154 | : [R_FPRS] "a" (__fprs) \ |
155 | input_regs \ |
156 | : "cc", "memory"); \ |
157 | __ret; \ |
158 | }) |
159 | |
160 | /* These builtins are usable in context of glibc lock elision code without any |
161 | changes. Use them. */ |
162 | #define __libc_tend() \ |
163 | ({ __asm__ __volatile__ (".machine push\n\t" \ |
164 | ".machinemode \"zarch_nohighgprs\"\n\t" \ |
165 | ".machine \"all\"\n\t"); \ |
166 | int __ret = __builtin_tend (); \ |
167 | __asm__ __volatile__ (".machine pop"); \ |
168 | __ret; \ |
169 | }) |
170 | |
171 | #define __libc_tabort(abortcode) \ |
172 | __asm__ __volatile__ (".machine push\n\t" \ |
173 | ".machinemode \"zarch_nohighgprs\"\n\t" \ |
174 | ".machine \"all\"\n\t"); \ |
175 | __builtin_tabort (abortcode); \ |
176 | __asm__ __volatile__ (".machine pop") |
177 | |
178 | #define __libc_tx_nesting_depth() \ |
179 | ({ __asm__ __volatile__ (".machine push\n\t" \ |
180 | ".machinemode \"zarch_nohighgprs\"\n\t" \ |
181 | ".machine \"all\"\n\t"); \ |
182 | int __ret = __builtin_tx_nesting_depth (); \ |
183 | __asm__ __volatile__ (".machine pop"); \ |
184 | __ret; \ |
185 | }) |
186 | |
187 | #endif |
188 | |