1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | |
3 | /* |
4 | * Copyright 2020 IBM Corp. |
5 | * |
6 | * Author: Bulent Abali <abali@us.ibm.com> |
7 | * |
8 | */ |
9 | #include <stdio.h> |
10 | #include <stdlib.h> |
11 | #include <string.h> |
12 | #include <unistd.h> |
13 | #include <stdint.h> |
14 | #include <sys/types.h> |
15 | #include <sys/stat.h> |
16 | #include <sys/time.h> |
17 | #include <sys/fcntl.h> |
18 | #include <sys/mman.h> |
19 | #include <endian.h> |
20 | #include <bits/endian.h> |
21 | #include <sys/ioctl.h> |
22 | #include <assert.h> |
23 | #include <errno.h> |
24 | #include <signal.h> |
25 | #include "vas-api.h" |
26 | #include "nx.h" |
27 | #include "copy-paste.h" |
28 | #include "nxu.h" |
29 | #include "nx_dbg.h" |
30 | #include <sys/platform/ppc.h> |
31 | |
32 | #define barrier() |
33 | #define hwsync() ({ asm volatile("sync" ::: "memory"); }) |
34 | |
35 | #ifndef NX_NO_CPU_PRI |
36 | #define cpu_pri_default() ({ asm volatile ("or 2, 2, 2"); }) |
37 | #define cpu_pri_low() ({ asm volatile ("or 31, 31, 31"); }) |
38 | #else |
39 | #define cpu_pri_default() |
40 | #define cpu_pri_low() |
41 | #endif |
42 | |
43 | void *nx_fault_storage_address; |
44 | |
45 | struct nx_handle { |
46 | int fd; |
47 | int function; |
48 | void *paste_addr; |
49 | }; |
50 | |
51 | static int open_device_nodes(char *devname, int pri, struct nx_handle *handle) |
52 | { |
53 | int rc, fd; |
54 | void *addr; |
55 | struct vas_tx_win_open_attr txattr; |
56 | |
57 | fd = open(devname, O_RDWR); |
58 | if (fd < 0) { |
59 | fprintf(stderr, " open device name %s\n" , devname); |
60 | return -errno; |
61 | } |
62 | |
63 | memset(&txattr, 0, sizeof(txattr)); |
64 | txattr.version = 1; |
65 | txattr.vas_id = pri; |
66 | rc = ioctl(fd, VAS_TX_WIN_OPEN, (unsigned long)&txattr); |
67 | if (rc < 0) { |
68 | fprintf(stderr, "ioctl() n %d, error %d\n" , rc, errno); |
69 | rc = -errno; |
70 | goto out; |
71 | } |
72 | |
73 | addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0ULL); |
74 | if (addr == MAP_FAILED) { |
75 | fprintf(stderr, "mmap() failed, errno %d\n" , errno); |
76 | rc = -errno; |
77 | goto out; |
78 | } |
79 | handle->fd = fd; |
80 | handle->paste_addr = (void *)((char *)addr + 0x400); |
81 | |
82 | rc = 0; |
83 | out: |
84 | close(fd); |
85 | return rc; |
86 | } |
87 | |
88 | void *nx_function_begin(int function, int pri) |
89 | { |
90 | int rc; |
91 | char *devname = "/dev/crypto/nx-gzip" ; |
92 | struct nx_handle *nxhandle; |
93 | |
94 | if (function != NX_FUNC_COMP_GZIP) { |
95 | errno = EINVAL; |
96 | fprintf(stderr, " NX_FUNC_COMP_GZIP not found\n" ); |
97 | return NULL; |
98 | } |
99 | |
100 | |
101 | nxhandle = malloc(sizeof(*nxhandle)); |
102 | if (!nxhandle) { |
103 | errno = ENOMEM; |
104 | fprintf(stderr, " No memory\n" ); |
105 | return NULL; |
106 | } |
107 | |
108 | nxhandle->function = function; |
109 | rc = open_device_nodes(devname, pri, handle: nxhandle); |
110 | if (rc < 0) { |
111 | errno = -rc; |
112 | fprintf(stderr, " open_device_nodes failed\n" ); |
113 | return NULL; |
114 | } |
115 | |
116 | return nxhandle; |
117 | } |
118 | |
119 | int nx_function_end(void *handle) |
120 | { |
121 | int rc = 0; |
122 | struct nx_handle *nxhandle = handle; |
123 | |
124 | rc = munmap(nxhandle->paste_addr - 0x400, 4096); |
125 | if (rc < 0) { |
126 | fprintf(stderr, "munmap() failed, errno %d\n" , errno); |
127 | return rc; |
128 | } |
129 | close(nxhandle->fd); |
130 | free(nxhandle); |
131 | |
132 | return rc; |
133 | } |
134 | |
135 | static int nx_wait_for_csb(struct nx_gzip_crb_cpb_t *cmdp) |
136 | { |
137 | long poll = 0; |
138 | uint64_t t; |
139 | |
140 | /* Save power and let other threads use the h/w. top may show |
141 | * 100% but only because OS doesn't know we slowed the this |
142 | * h/w thread while polling. We're letting other threads have |
143 | * higher throughput on the core. |
144 | */ |
145 | cpu_pri_low(); |
146 | |
147 | #define CSB_MAX_POLL 200000000UL |
148 | #define USLEEP_TH 300000UL |
149 | |
150 | t = __ppc_get_timebase(); |
151 | |
152 | while (getnn(cmdp->crb.csb, csb_v) == 0) { |
153 | ++poll; |
154 | hwsync(); |
155 | |
156 | cpu_pri_low(); |
157 | |
158 | /* usleep(0) takes around 29000 ticks ~60 us. |
159 | * 300000 is spinning for about 600 us then |
160 | * start sleeping. |
161 | */ |
162 | if ((__ppc_get_timebase() - t) > USLEEP_TH) { |
163 | cpu_pri_default(); |
164 | usleep(1); |
165 | } |
166 | |
167 | if (poll > CSB_MAX_POLL) |
168 | break; |
169 | |
170 | /* Fault address from signal handler */ |
171 | if (nx_fault_storage_address) { |
172 | cpu_pri_default(); |
173 | return -EAGAIN; |
174 | } |
175 | |
176 | } |
177 | |
178 | cpu_pri_default(); |
179 | |
180 | /* hw has updated csb and output buffer */ |
181 | hwsync(); |
182 | |
183 | /* Check CSB flags. */ |
184 | if (getnn(cmdp->crb.csb, csb_v) == 0) { |
185 | fprintf(stderr, "CSB still not valid after %d polls.\n" , |
186 | (int) poll); |
187 | prt_err("CSB still not valid after %d polls, giving up.\n" , |
188 | (int) poll); |
189 | return -ETIMEDOUT; |
190 | } |
191 | |
192 | return 0; |
193 | } |
194 | |
195 | static int nxu_run_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle) |
196 | { |
197 | int i, ret, retries; |
198 | struct nx_handle *nxhandle = handle; |
199 | |
200 | assert(handle != NULL); |
201 | i = 0; |
202 | retries = 5000; |
203 | while (i++ < retries) { |
204 | hwsync(); |
205 | vas_copy(&cmdp->crb, 0); |
206 | ret = vas_paste(nxhandle->paste_addr, 0); |
207 | hwsync(); |
208 | |
209 | NXPRT(fprintf(stderr, "Paste attempt %d/%d returns 0x%x\n" , |
210 | i, retries, ret)); |
211 | |
212 | if ((ret == 2) || (ret == 3)) { |
213 | |
214 | ret = nx_wait_for_csb(cmdp); |
215 | if (!ret) { |
216 | goto out; |
217 | } else if (ret == -EAGAIN) { |
218 | long x; |
219 | |
220 | prt_err("Touching address %p, 0x%lx\n" , |
221 | nx_fault_storage_address, |
222 | *(long *) nx_fault_storage_address); |
223 | x = *(long *) nx_fault_storage_address; |
224 | *(long *) nx_fault_storage_address = x; |
225 | nx_fault_storage_address = 0; |
226 | continue; |
227 | } else { |
228 | prt_err("wait_for_csb() returns %d\n" , ret); |
229 | break; |
230 | } |
231 | } else { |
232 | if (i < 10) { |
233 | /* spin for few ticks */ |
234 | #define SPIN_TH 500UL |
235 | uint64_t fail_spin; |
236 | |
237 | fail_spin = __ppc_get_timebase(); |
238 | while ((__ppc_get_timebase() - fail_spin) < |
239 | SPIN_TH) |
240 | ; |
241 | } else { |
242 | /* sleep */ |
243 | unsigned int pr = 0; |
244 | |
245 | if (pr++ % 100 == 0) { |
246 | prt_err("Paste attempt %d/" , i); |
247 | prt_err("%d, failed pid= %d\n" , retries, |
248 | getpid()); |
249 | } |
250 | usleep(1); |
251 | } |
252 | continue; |
253 | } |
254 | } |
255 | |
256 | out: |
257 | cpu_pri_default(); |
258 | |
259 | return ret; |
260 | } |
261 | |
262 | int nxu_submit_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle) |
263 | { |
264 | int cc; |
265 | |
266 | cc = nxu_run_job(cmdp, handle); |
267 | |
268 | if (!cc) |
269 | cc = getnn(cmdp->crb.csb, csb_cc); /* CC Table 6-8 */ |
270 | |
271 | return cc; |
272 | } |
273 | |
274 | |
275 | void nxu_sigsegv_handler(int sig, siginfo_t *info, void *ctx) |
276 | { |
277 | fprintf(stderr, "%d: Got signal %d si_code %d, si_addr %p\n" , getpid(), |
278 | sig, info->si_code, info->si_addr); |
279 | |
280 | nx_fault_storage_address = info->si_addr; |
281 | } |
282 | |
283 | /* |
284 | * Fault in pages prior to NX job submission. wr=1 may be required to |
285 | * touch writeable pages. System zero pages do not fault-in the page as |
286 | * intended. Typically set wr=1 for NX target pages and set wr=0 for NX |
287 | * source pages. |
288 | */ |
289 | int nxu_touch_pages(void *buf, long buf_len, long page_len, int wr) |
290 | { |
291 | char *begin = buf; |
292 | char *end = (char *) buf + buf_len - 1; |
293 | volatile char t; |
294 | |
295 | assert(buf_len >= 0 && !!buf); |
296 | |
297 | NXPRT(fprintf(stderr, "touch %p %p len 0x%lx wr=%d\n" , buf, |
298 | (buf + buf_len), buf_len, wr)); |
299 | |
300 | if (buf_len <= 0 || buf == NULL) |
301 | return -1; |
302 | |
303 | do { |
304 | t = *begin; |
305 | if (wr) |
306 | *begin = t; |
307 | begin = begin + page_len; |
308 | } while (begin < end); |
309 | |
310 | /* When buf_sz is small or buf tail is in another page */ |
311 | t = *end; |
312 | if (wr) |
313 | *end = t; |
314 | |
315 | return 0; |
316 | } |
317 | |