1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * linux/net/sunrpc/svcsock.c |
4 | * |
5 | * These are the RPC server socket internals. |
6 | * |
7 | * The server scheduling algorithm does not always distribute the load |
8 | * evenly when servicing a single client. May need to modify the |
9 | * svc_xprt_enqueue procedure... |
10 | * |
11 | * TCP support is largely untested and may be a little slow. The problem |
12 | * is that we currently do two separate recvfrom's, one for the 4-byte |
13 | * record length, and the second for the actual record. This could possibly |
14 | * be improved by always reading a minimum size of around 100 bytes and |
15 | * tucking any superfluous bytes away in a temporary store. Still, that |
16 | * leaves write requests out in the rain. An alternative may be to peek at |
17 | * the first skb in the queue, and if it matches the next TCP sequence |
18 | * number, to extract the record marker. Yuck. |
19 | * |
20 | * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de> |
21 | */ |
22 | |
23 | #include <linux/kernel.h> |
24 | #include <linux/sched.h> |
25 | #include <linux/module.h> |
26 | #include <linux/errno.h> |
27 | #include <linux/fcntl.h> |
28 | #include <linux/net.h> |
29 | #include <linux/in.h> |
30 | #include <linux/inet.h> |
31 | #include <linux/udp.h> |
32 | #include <linux/tcp.h> |
33 | #include <linux/unistd.h> |
34 | #include <linux/slab.h> |
35 | #include <linux/netdevice.h> |
36 | #include <linux/skbuff.h> |
37 | #include <linux/file.h> |
38 | #include <linux/freezer.h> |
39 | #include <linux/bvec.h> |
40 | |
41 | #include <net/sock.h> |
42 | #include <net/checksum.h> |
43 | #include <net/ip.h> |
44 | #include <net/ipv6.h> |
45 | #include <net/udp.h> |
46 | #include <net/tcp.h> |
47 | #include <net/tcp_states.h> |
48 | #include <net/tls_prot.h> |
49 | #include <net/handshake.h> |
50 | #include <linux/uaccess.h> |
51 | #include <linux/highmem.h> |
52 | #include <asm/ioctls.h> |
53 | #include <linux/key.h> |
54 | |
55 | #include <linux/sunrpc/types.h> |
56 | #include <linux/sunrpc/clnt.h> |
57 | #include <linux/sunrpc/xdr.h> |
58 | #include <linux/sunrpc/msg_prot.h> |
59 | #include <linux/sunrpc/svcsock.h> |
60 | #include <linux/sunrpc/stats.h> |
61 | #include <linux/sunrpc/xprt.h> |
62 | |
63 | #include <trace/events/sock.h> |
64 | #include <trace/events/sunrpc.h> |
65 | |
66 | #include "socklib.h" |
67 | #include "sunrpc.h" |
68 | |
69 | #define RPCDBG_FACILITY RPCDBG_SVCXPRT |
70 | |
71 | /* To-do: to avoid tying up an nfsd thread while waiting for a |
72 | * handshake request, the request could instead be deferred. |
73 | */ |
74 | enum { |
75 | SVC_HANDSHAKE_TO = 5U * HZ |
76 | }; |
77 | |
78 | static struct svc_sock *svc_setup_socket(struct svc_serv *, struct socket *, |
79 | int flags); |
80 | static int svc_udp_recvfrom(struct svc_rqst *); |
81 | static int svc_udp_sendto(struct svc_rqst *); |
82 | static void svc_sock_detach(struct svc_xprt *); |
83 | static void svc_tcp_sock_detach(struct svc_xprt *); |
84 | static void svc_sock_free(struct svc_xprt *); |
85 | |
86 | static struct svc_xprt *svc_create_socket(struct svc_serv *, int, |
87 | struct net *, struct sockaddr *, |
88 | int, int); |
89 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
90 | static struct lock_class_key svc_key[2]; |
91 | static struct lock_class_key svc_slock_key[2]; |
92 | |
93 | static void svc_reclassify_socket(struct socket *sock) |
94 | { |
95 | struct sock *sk = sock->sk; |
96 | |
97 | if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) |
98 | return; |
99 | |
100 | switch (sk->sk_family) { |
101 | case AF_INET: |
102 | sock_lock_init_class_and_name(sk, "slock-AF_INET-NFSD" , |
103 | &svc_slock_key[0], |
104 | "sk_xprt.xpt_lock-AF_INET-NFSD" , |
105 | &svc_key[0]); |
106 | break; |
107 | |
108 | case AF_INET6: |
109 | sock_lock_init_class_and_name(sk, "slock-AF_INET6-NFSD" , |
110 | &svc_slock_key[1], |
111 | "sk_xprt.xpt_lock-AF_INET6-NFSD" , |
112 | &svc_key[1]); |
113 | break; |
114 | |
115 | default: |
116 | BUG(); |
117 | } |
118 | } |
119 | #else |
120 | static void svc_reclassify_socket(struct socket *sock) |
121 | { |
122 | } |
123 | #endif |
124 | |
125 | /** |
126 | * svc_tcp_release_ctxt - Release transport-related resources |
127 | * @xprt: the transport which owned the context |
128 | * @ctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt |
129 | * |
130 | */ |
131 | static void svc_tcp_release_ctxt(struct svc_xprt *xprt, void *ctxt) |
132 | { |
133 | } |
134 | |
135 | /** |
136 | * svc_udp_release_ctxt - Release transport-related resources |
137 | * @xprt: the transport which owned the context |
138 | * @ctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt |
139 | * |
140 | */ |
141 | static void svc_udp_release_ctxt(struct svc_xprt *xprt, void *ctxt) |
142 | { |
143 | struct sk_buff *skb = ctxt; |
144 | |
145 | if (skb) |
146 | consume_skb(skb); |
147 | } |
148 | |
149 | union svc_pktinfo_u { |
150 | struct in_pktinfo pkti; |
151 | struct in6_pktinfo pkti6; |
152 | }; |
153 | #define SVC_PKTINFO_SPACE \ |
154 | CMSG_SPACE(sizeof(union svc_pktinfo_u)) |
155 | |
156 | static void svc_set_cmsg_data(struct svc_rqst *rqstp, struct cmsghdr *cmh) |
157 | { |
158 | struct svc_sock *svsk = |
159 | container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); |
160 | switch (svsk->sk_sk->sk_family) { |
161 | case AF_INET: { |
162 | struct in_pktinfo *pki = CMSG_DATA(cmh); |
163 | |
164 | cmh->cmsg_level = SOL_IP; |
165 | cmh->cmsg_type = IP_PKTINFO; |
166 | pki->ipi_ifindex = 0; |
167 | pki->ipi_spec_dst.s_addr = |
168 | svc_daddr_in(rqst: rqstp)->sin_addr.s_addr; |
169 | cmh->cmsg_len = CMSG_LEN(sizeof(*pki)); |
170 | } |
171 | break; |
172 | |
173 | case AF_INET6: { |
174 | struct in6_pktinfo *pki = CMSG_DATA(cmh); |
175 | struct sockaddr_in6 *daddr = svc_daddr_in6(rqst: rqstp); |
176 | |
177 | cmh->cmsg_level = SOL_IPV6; |
178 | cmh->cmsg_type = IPV6_PKTINFO; |
179 | pki->ipi6_ifindex = daddr->sin6_scope_id; |
180 | pki->ipi6_addr = daddr->sin6_addr; |
181 | cmh->cmsg_len = CMSG_LEN(sizeof(*pki)); |
182 | } |
183 | break; |
184 | } |
185 | } |
186 | |
187 | static int svc_sock_result_payload(struct svc_rqst *rqstp, unsigned int offset, |
188 | unsigned int length) |
189 | { |
190 | return 0; |
191 | } |
192 | |
193 | /* |
194 | * Report socket names for nfsdfs |
195 | */ |
196 | static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining) |
197 | { |
198 | const struct sock *sk = svsk->sk_sk; |
199 | const char *proto_name = sk->sk_protocol == IPPROTO_UDP ? |
200 | "udp" : "tcp" ; |
201 | int len; |
202 | |
203 | switch (sk->sk_family) { |
204 | case PF_INET: |
205 | len = snprintf(buf, size: remaining, fmt: "ipv4 %s %pI4 %d\n" , |
206 | proto_name, |
207 | &inet_sk(sk)->inet_rcv_saddr, |
208 | inet_sk(sk)->inet_num); |
209 | break; |
210 | #if IS_ENABLED(CONFIG_IPV6) |
211 | case PF_INET6: |
212 | len = snprintf(buf, size: remaining, fmt: "ipv6 %s %pI6 %d\n" , |
213 | proto_name, |
214 | &sk->sk_v6_rcv_saddr, |
215 | inet_sk(sk)->inet_num); |
216 | break; |
217 | #endif |
218 | default: |
219 | len = snprintf(buf, size: remaining, fmt: "*unknown-%d*\n" , |
220 | sk->sk_family); |
221 | } |
222 | |
223 | if (len >= remaining) { |
224 | *buf = '\0'; |
225 | return -ENAMETOOLONG; |
226 | } |
227 | return len; |
228 | } |
229 | |
230 | static int |
231 | svc_tcp_sock_process_cmsg(struct socket *sock, struct msghdr *msg, |
232 | struct cmsghdr *cmsg, int ret) |
233 | { |
234 | u8 content_type = tls_get_record_type(sk: sock->sk, msg: cmsg); |
235 | u8 level, description; |
236 | |
237 | switch (content_type) { |
238 | case 0: |
239 | break; |
240 | case TLS_RECORD_TYPE_DATA: |
241 | /* TLS sets EOR at the end of each application data |
242 | * record, even though there might be more frames |
243 | * waiting to be decrypted. |
244 | */ |
245 | msg->msg_flags &= ~MSG_EOR; |
246 | break; |
247 | case TLS_RECORD_TYPE_ALERT: |
248 | tls_alert_recv(sk: sock->sk, msg, level: &level, description: &description); |
249 | ret = (level == TLS_ALERT_LEVEL_FATAL) ? |
250 | -ENOTCONN : -EAGAIN; |
251 | break; |
252 | default: |
253 | /* discard this record type */ |
254 | ret = -EAGAIN; |
255 | } |
256 | return ret; |
257 | } |
258 | |
259 | static int |
260 | svc_tcp_sock_recv_cmsg(struct svc_sock *svsk, struct msghdr *msg) |
261 | { |
262 | union { |
263 | struct cmsghdr cmsg; |
264 | u8 buf[CMSG_SPACE(sizeof(u8))]; |
265 | } u; |
266 | struct socket *sock = svsk->sk_sock; |
267 | int ret; |
268 | |
269 | msg->msg_control = &u; |
270 | msg->msg_controllen = sizeof(u); |
271 | ret = sock_recvmsg(sock, msg, MSG_DONTWAIT); |
272 | if (unlikely(msg->msg_controllen != sizeof(u))) |
273 | ret = svc_tcp_sock_process_cmsg(sock, msg, cmsg: &u.cmsg, ret); |
274 | return ret; |
275 | } |
276 | |
277 | #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE |
278 | static void svc_flush_bvec(const struct bio_vec *bvec, size_t size, size_t seek) |
279 | { |
280 | struct bvec_iter bi = { |
281 | .bi_size = size + seek, |
282 | }; |
283 | struct bio_vec bv; |
284 | |
285 | bvec_iter_advance(bvec, &bi, seek & PAGE_MASK); |
286 | for_each_bvec(bv, bvec, bi, bi) |
287 | flush_dcache_page(bv.bv_page); |
288 | } |
289 | #else |
290 | static inline void svc_flush_bvec(const struct bio_vec *bvec, size_t size, |
291 | size_t seek) |
292 | { |
293 | } |
294 | #endif |
295 | |
296 | /* |
297 | * Read from @rqstp's transport socket. The incoming message fills whole |
298 | * pages in @rqstp's rq_pages array until the last page of the message |
299 | * has been received into a partial page. |
300 | */ |
301 | static ssize_t svc_tcp_read_msg(struct svc_rqst *rqstp, size_t buflen, |
302 | size_t seek) |
303 | { |
304 | struct svc_sock *svsk = |
305 | container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); |
306 | struct bio_vec *bvec = rqstp->rq_bvec; |
307 | struct msghdr msg = { NULL }; |
308 | unsigned int i; |
309 | ssize_t len; |
310 | size_t t; |
311 | |
312 | clear_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags); |
313 | |
314 | for (i = 0, t = 0; t < buflen; i++, t += PAGE_SIZE) |
315 | bvec_set_page(bv: &bvec[i], page: rqstp->rq_pages[i], PAGE_SIZE, offset: 0); |
316 | rqstp->rq_respages = &rqstp->rq_pages[i]; |
317 | rqstp->rq_next_page = rqstp->rq_respages + 1; |
318 | |
319 | iov_iter_bvec(i: &msg.msg_iter, ITER_DEST, bvec, nr_segs: i, count: buflen); |
320 | if (seek) { |
321 | iov_iter_advance(i: &msg.msg_iter, bytes: seek); |
322 | buflen -= seek; |
323 | } |
324 | len = svc_tcp_sock_recv_cmsg(svsk, msg: &msg); |
325 | if (len > 0) |
326 | svc_flush_bvec(bvec, size: len, seek); |
327 | |
328 | /* If we read a full record, then assume there may be more |
329 | * data to read (stream based sockets only!) |
330 | */ |
331 | if (len == buflen) |
332 | set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags); |
333 | |
334 | return len; |
335 | } |
336 | |
337 | /* |
338 | * Set socket snd and rcv buffer lengths |
339 | */ |
340 | static void svc_sock_setbufsize(struct svc_sock *svsk, unsigned int nreqs) |
341 | { |
342 | unsigned int max_mesg = svsk->sk_xprt.xpt_server->sv_max_mesg; |
343 | struct socket *sock = svsk->sk_sock; |
344 | |
345 | nreqs = min(nreqs, INT_MAX / 2 / max_mesg); |
346 | |
347 | lock_sock(sk: sock->sk); |
348 | sock->sk->sk_sndbuf = nreqs * max_mesg * 2; |
349 | sock->sk->sk_rcvbuf = nreqs * max_mesg * 2; |
350 | sock->sk->sk_write_space(sock->sk); |
351 | release_sock(sk: sock->sk); |
352 | } |
353 | |
354 | static void svc_sock_secure_port(struct svc_rqst *rqstp) |
355 | { |
356 | if (svc_port_is_privileged(sin: svc_addr(rqst: rqstp))) |
357 | set_bit(nr: RQ_SECURE, addr: &rqstp->rq_flags); |
358 | else |
359 | clear_bit(nr: RQ_SECURE, addr: &rqstp->rq_flags); |
360 | } |
361 | |
362 | /* |
363 | * INET callback when data has been received on the socket. |
364 | */ |
365 | static void svc_data_ready(struct sock *sk) |
366 | { |
367 | struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; |
368 | |
369 | trace_sk_data_ready(sk); |
370 | |
371 | if (svsk) { |
372 | /* Refer to svc_setup_socket() for details. */ |
373 | rmb(); |
374 | svsk->sk_odata(sk); |
375 | trace_svcsock_data_ready(xprt: &svsk->sk_xprt, result: 0); |
376 | if (test_bit(XPT_HANDSHAKE, &svsk->sk_xprt.xpt_flags)) |
377 | return; |
378 | if (!test_and_set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags)) |
379 | svc_xprt_enqueue(xprt: &svsk->sk_xprt); |
380 | } |
381 | } |
382 | |
383 | /* |
384 | * INET callback when space is newly available on the socket. |
385 | */ |
386 | static void svc_write_space(struct sock *sk) |
387 | { |
388 | struct svc_sock *svsk = (struct svc_sock *)(sk->sk_user_data); |
389 | |
390 | if (svsk) { |
391 | /* Refer to svc_setup_socket() for details. */ |
392 | rmb(); |
393 | trace_svcsock_write_space(xprt: &svsk->sk_xprt, result: 0); |
394 | svsk->sk_owspace(sk); |
395 | svc_xprt_enqueue(xprt: &svsk->sk_xprt); |
396 | } |
397 | } |
398 | |
399 | static int svc_tcp_has_wspace(struct svc_xprt *xprt) |
400 | { |
401 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
402 | |
403 | if (test_bit(XPT_LISTENER, &xprt->xpt_flags)) |
404 | return 1; |
405 | return !test_bit(SOCK_NOSPACE, &svsk->sk_sock->flags); |
406 | } |
407 | |
408 | static void svc_tcp_kill_temp_xprt(struct svc_xprt *xprt) |
409 | { |
410 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
411 | |
412 | sock_no_linger(sk: svsk->sk_sock->sk); |
413 | } |
414 | |
415 | /** |
416 | * svc_tcp_handshake_done - Handshake completion handler |
417 | * @data: address of xprt to wake |
418 | * @status: status of handshake |
419 | * @peerid: serial number of key containing the remote peer's identity |
420 | * |
421 | * If a security policy is specified as an export option, we don't |
422 | * have a specific export here to check. So we set a "TLS session |
423 | * is present" flag on the xprt and let an upper layer enforce local |
424 | * security policy. |
425 | */ |
426 | static void svc_tcp_handshake_done(void *data, int status, key_serial_t peerid) |
427 | { |
428 | struct svc_xprt *xprt = data; |
429 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
430 | |
431 | if (!status) { |
432 | if (peerid != TLS_NO_PEERID) |
433 | set_bit(nr: XPT_PEER_AUTH, addr: &xprt->xpt_flags); |
434 | set_bit(nr: XPT_TLS_SESSION, addr: &xprt->xpt_flags); |
435 | } |
436 | clear_bit(nr: XPT_HANDSHAKE, addr: &xprt->xpt_flags); |
437 | complete_all(&svsk->sk_handshake_done); |
438 | } |
439 | |
440 | /** |
441 | * svc_tcp_handshake - Perform a transport-layer security handshake |
442 | * @xprt: connected transport endpoint |
443 | * |
444 | */ |
445 | static void svc_tcp_handshake(struct svc_xprt *xprt) |
446 | { |
447 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
448 | struct sock *sk = svsk->sk_sock->sk; |
449 | struct tls_handshake_args args = { |
450 | .ta_sock = svsk->sk_sock, |
451 | .ta_done = svc_tcp_handshake_done, |
452 | .ta_data = xprt, |
453 | }; |
454 | int ret; |
455 | |
456 | trace_svc_tls_upcall(xprt); |
457 | |
458 | clear_bit(nr: XPT_TLS_SESSION, addr: &xprt->xpt_flags); |
459 | init_completion(x: &svsk->sk_handshake_done); |
460 | |
461 | ret = tls_server_hello_x509(args: &args, GFP_KERNEL); |
462 | if (ret) { |
463 | trace_svc_tls_not_started(xprt); |
464 | goto out_failed; |
465 | } |
466 | |
467 | ret = wait_for_completion_interruptible_timeout(x: &svsk->sk_handshake_done, |
468 | timeout: SVC_HANDSHAKE_TO); |
469 | if (ret <= 0) { |
470 | if (tls_handshake_cancel(sk)) { |
471 | trace_svc_tls_timed_out(xprt); |
472 | goto out_close; |
473 | } |
474 | } |
475 | |
476 | if (!test_bit(XPT_TLS_SESSION, &xprt->xpt_flags)) { |
477 | trace_svc_tls_unavailable(xprt); |
478 | goto out_close; |
479 | } |
480 | |
481 | /* Mark the transport ready in case the remote sent RPC |
482 | * traffic before the kernel received the handshake |
483 | * completion downcall. |
484 | */ |
485 | set_bit(nr: XPT_DATA, addr: &xprt->xpt_flags); |
486 | svc_xprt_enqueue(xprt); |
487 | return; |
488 | |
489 | out_close: |
490 | set_bit(nr: XPT_CLOSE, addr: &xprt->xpt_flags); |
491 | out_failed: |
492 | clear_bit(nr: XPT_HANDSHAKE, addr: &xprt->xpt_flags); |
493 | set_bit(nr: XPT_DATA, addr: &xprt->xpt_flags); |
494 | svc_xprt_enqueue(xprt); |
495 | } |
496 | |
497 | /* |
498 | * See net/ipv6/ip_sockglue.c : ip_cmsg_recv_pktinfo |
499 | */ |
500 | static int svc_udp_get_dest_address4(struct svc_rqst *rqstp, |
501 | struct cmsghdr *cmh) |
502 | { |
503 | struct in_pktinfo *pki = CMSG_DATA(cmh); |
504 | struct sockaddr_in *daddr = svc_daddr_in(rqst: rqstp); |
505 | |
506 | if (cmh->cmsg_type != IP_PKTINFO) |
507 | return 0; |
508 | |
509 | daddr->sin_family = AF_INET; |
510 | daddr->sin_addr.s_addr = pki->ipi_spec_dst.s_addr; |
511 | return 1; |
512 | } |
513 | |
514 | /* |
515 | * See net/ipv6/datagram.c : ip6_datagram_recv_ctl |
516 | */ |
517 | static int svc_udp_get_dest_address6(struct svc_rqst *rqstp, |
518 | struct cmsghdr *cmh) |
519 | { |
520 | struct in6_pktinfo *pki = CMSG_DATA(cmh); |
521 | struct sockaddr_in6 *daddr = svc_daddr_in6(rqst: rqstp); |
522 | |
523 | if (cmh->cmsg_type != IPV6_PKTINFO) |
524 | return 0; |
525 | |
526 | daddr->sin6_family = AF_INET6; |
527 | daddr->sin6_addr = pki->ipi6_addr; |
528 | daddr->sin6_scope_id = pki->ipi6_ifindex; |
529 | return 1; |
530 | } |
531 | |
532 | /* |
533 | * Copy the UDP datagram's destination address to the rqstp structure. |
534 | * The 'destination' address in this case is the address to which the |
535 | * peer sent the datagram, i.e. our local address. For multihomed |
536 | * hosts, this can change from msg to msg. Note that only the IP |
537 | * address changes, the port number should remain the same. |
538 | */ |
539 | static int svc_udp_get_dest_address(struct svc_rqst *rqstp, |
540 | struct cmsghdr *cmh) |
541 | { |
542 | switch (cmh->cmsg_level) { |
543 | case SOL_IP: |
544 | return svc_udp_get_dest_address4(rqstp, cmh); |
545 | case SOL_IPV6: |
546 | return svc_udp_get_dest_address6(rqstp, cmh); |
547 | } |
548 | |
549 | return 0; |
550 | } |
551 | |
552 | /** |
553 | * svc_udp_recvfrom - Receive a datagram from a UDP socket. |
554 | * @rqstp: request structure into which to receive an RPC Call |
555 | * |
556 | * Called in a loop when XPT_DATA has been set. |
557 | * |
558 | * Returns: |
559 | * On success, the number of bytes in a received RPC Call, or |
560 | * %0 if a complete RPC Call message was not ready to return |
561 | */ |
562 | static int svc_udp_recvfrom(struct svc_rqst *rqstp) |
563 | { |
564 | struct svc_sock *svsk = |
565 | container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); |
566 | struct svc_serv *serv = svsk->sk_xprt.xpt_server; |
567 | struct sk_buff *skb; |
568 | union { |
569 | struct cmsghdr hdr; |
570 | long all[SVC_PKTINFO_SPACE / sizeof(long)]; |
571 | } buffer; |
572 | struct cmsghdr *cmh = &buffer.hdr; |
573 | struct msghdr msg = { |
574 | .msg_name = svc_addr(rqst: rqstp), |
575 | .msg_control = cmh, |
576 | .msg_controllen = sizeof(buffer), |
577 | .msg_flags = MSG_DONTWAIT, |
578 | }; |
579 | size_t len; |
580 | int err; |
581 | |
582 | if (test_and_clear_bit(nr: XPT_CHNGBUF, addr: &svsk->sk_xprt.xpt_flags)) |
583 | /* udp sockets need large rcvbuf as all pending |
584 | * requests are still in that buffer. sndbuf must |
585 | * also be large enough that there is enough space |
586 | * for one reply per thread. We count all threads |
587 | * rather than threads in a particular pool, which |
588 | * provides an upper bound on the number of threads |
589 | * which will access the socket. |
590 | */ |
591 | svc_sock_setbufsize(svsk, nreqs: serv->sv_nrthreads + 3); |
592 | |
593 | clear_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags); |
594 | err = kernel_recvmsg(sock: svsk->sk_sock, msg: &msg, NULL, |
595 | num: 0, len: 0, MSG_PEEK | MSG_DONTWAIT); |
596 | if (err < 0) |
597 | goto out_recv_err; |
598 | skb = skb_recv_udp(sk: svsk->sk_sk, MSG_DONTWAIT, err: &err); |
599 | if (!skb) |
600 | goto out_recv_err; |
601 | |
602 | len = svc_addr_len(sa: svc_addr(rqst: rqstp)); |
603 | rqstp->rq_addrlen = len; |
604 | if (skb->tstamp == 0) { |
605 | skb->tstamp = ktime_get_real(); |
606 | /* Don't enable netstamp, sunrpc doesn't |
607 | need that much accuracy */ |
608 | } |
609 | sock_write_timestamp(sk: svsk->sk_sk, kt: skb->tstamp); |
610 | set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags); /* there may be more data... */ |
611 | |
612 | len = skb->len; |
613 | rqstp->rq_arg.len = len; |
614 | trace_svcsock_udp_recv(xprt: &svsk->sk_xprt, result: len); |
615 | |
616 | rqstp->rq_prot = IPPROTO_UDP; |
617 | |
618 | if (!svc_udp_get_dest_address(rqstp, cmh)) |
619 | goto out_cmsg_err; |
620 | rqstp->rq_daddrlen = svc_addr_len(sa: svc_daddr(rqst: rqstp)); |
621 | |
622 | if (skb_is_nonlinear(skb)) { |
623 | /* we have to copy */ |
624 | local_bh_disable(); |
625 | if (csum_partial_copy_to_xdr(xdr: &rqstp->rq_arg, skb)) |
626 | goto out_bh_enable; |
627 | local_bh_enable(); |
628 | consume_skb(skb); |
629 | } else { |
630 | /* we can use it in-place */ |
631 | rqstp->rq_arg.head[0].iov_base = skb->data; |
632 | rqstp->rq_arg.head[0].iov_len = len; |
633 | if (skb_checksum_complete(skb)) |
634 | goto out_free; |
635 | rqstp->rq_xprt_ctxt = skb; |
636 | } |
637 | |
638 | rqstp->rq_arg.page_base = 0; |
639 | if (len <= rqstp->rq_arg.head[0].iov_len) { |
640 | rqstp->rq_arg.head[0].iov_len = len; |
641 | rqstp->rq_arg.page_len = 0; |
642 | rqstp->rq_respages = rqstp->rq_pages+1; |
643 | } else { |
644 | rqstp->rq_arg.page_len = len - rqstp->rq_arg.head[0].iov_len; |
645 | rqstp->rq_respages = rqstp->rq_pages + 1 + |
646 | DIV_ROUND_UP(rqstp->rq_arg.page_len, PAGE_SIZE); |
647 | } |
648 | rqstp->rq_next_page = rqstp->rq_respages+1; |
649 | |
650 | if (serv->sv_stats) |
651 | serv->sv_stats->netudpcnt++; |
652 | |
653 | svc_sock_secure_port(rqstp); |
654 | svc_xprt_received(xprt: rqstp->rq_xprt); |
655 | return len; |
656 | |
657 | out_recv_err: |
658 | if (err != -EAGAIN) { |
659 | /* possibly an icmp error */ |
660 | set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags); |
661 | } |
662 | trace_svcsock_udp_recv_err(xprt: &svsk->sk_xprt, result: err); |
663 | goto out_clear_busy; |
664 | out_cmsg_err: |
665 | net_warn_ratelimited("svc: received unknown control message %d/%d; dropping RPC reply datagram\n" , |
666 | cmh->cmsg_level, cmh->cmsg_type); |
667 | goto out_free; |
668 | out_bh_enable: |
669 | local_bh_enable(); |
670 | out_free: |
671 | kfree_skb(skb); |
672 | out_clear_busy: |
673 | svc_xprt_received(xprt: rqstp->rq_xprt); |
674 | return 0; |
675 | } |
676 | |
677 | /** |
678 | * svc_udp_sendto - Send out a reply on a UDP socket |
679 | * @rqstp: completed svc_rqst |
680 | * |
681 | * xpt_mutex ensures @rqstp's whole message is written to the socket |
682 | * without interruption. |
683 | * |
684 | * Returns the number of bytes sent, or a negative errno. |
685 | */ |
686 | static int svc_udp_sendto(struct svc_rqst *rqstp) |
687 | { |
688 | struct svc_xprt *xprt = rqstp->rq_xprt; |
689 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
690 | struct xdr_buf *xdr = &rqstp->rq_res; |
691 | union { |
692 | struct cmsghdr hdr; |
693 | long all[SVC_PKTINFO_SPACE / sizeof(long)]; |
694 | } buffer; |
695 | struct cmsghdr *cmh = &buffer.hdr; |
696 | struct msghdr msg = { |
697 | .msg_name = &rqstp->rq_addr, |
698 | .msg_namelen = rqstp->rq_addrlen, |
699 | .msg_control = cmh, |
700 | .msg_flags = MSG_SPLICE_PAGES, |
701 | .msg_controllen = sizeof(buffer), |
702 | }; |
703 | unsigned int count; |
704 | int err; |
705 | |
706 | svc_udp_release_ctxt(xprt, ctxt: rqstp->rq_xprt_ctxt); |
707 | rqstp->rq_xprt_ctxt = NULL; |
708 | |
709 | svc_set_cmsg_data(rqstp, cmh); |
710 | |
711 | mutex_lock(&xprt->xpt_mutex); |
712 | |
713 | if (svc_xprt_is_dead(xprt)) |
714 | goto out_notconn; |
715 | |
716 | count = xdr_buf_to_bvec(bvec: rqstp->rq_bvec, |
717 | ARRAY_SIZE(rqstp->rq_bvec), xdr); |
718 | |
719 | iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: rqstp->rq_bvec, |
720 | nr_segs: count, count: 0); |
721 | err = sock_sendmsg(sock: svsk->sk_sock, msg: &msg); |
722 | if (err == -ECONNREFUSED) { |
723 | /* ICMP error on earlier request. */ |
724 | iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: rqstp->rq_bvec, |
725 | nr_segs: count, count: 0); |
726 | err = sock_sendmsg(sock: svsk->sk_sock, msg: &msg); |
727 | } |
728 | |
729 | trace_svcsock_udp_send(xprt, result: err); |
730 | |
731 | mutex_unlock(lock: &xprt->xpt_mutex); |
732 | return err; |
733 | |
734 | out_notconn: |
735 | mutex_unlock(lock: &xprt->xpt_mutex); |
736 | return -ENOTCONN; |
737 | } |
738 | |
739 | static int svc_udp_has_wspace(struct svc_xprt *xprt) |
740 | { |
741 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
742 | struct svc_serv *serv = xprt->xpt_server; |
743 | unsigned long required; |
744 | |
745 | /* |
746 | * Set the SOCK_NOSPACE flag before checking the available |
747 | * sock space. |
748 | */ |
749 | set_bit(SOCK_NOSPACE, addr: &svsk->sk_sock->flags); |
750 | required = atomic_read(v: &svsk->sk_xprt.xpt_reserved) + serv->sv_max_mesg; |
751 | if (required*2 > sock_wspace(sk: svsk->sk_sk)) |
752 | return 0; |
753 | clear_bit(SOCK_NOSPACE, addr: &svsk->sk_sock->flags); |
754 | return 1; |
755 | } |
756 | |
757 | static struct svc_xprt *svc_udp_accept(struct svc_xprt *xprt) |
758 | { |
759 | BUG(); |
760 | return NULL; |
761 | } |
762 | |
763 | static void svc_udp_kill_temp_xprt(struct svc_xprt *xprt) |
764 | { |
765 | } |
766 | |
767 | static struct svc_xprt *svc_udp_create(struct svc_serv *serv, |
768 | struct net *net, |
769 | struct sockaddr *sa, int salen, |
770 | int flags) |
771 | { |
772 | return svc_create_socket(serv, IPPROTO_UDP, net, sa, salen, flags); |
773 | } |
774 | |
775 | static const struct svc_xprt_ops svc_udp_ops = { |
776 | .xpo_create = svc_udp_create, |
777 | .xpo_recvfrom = svc_udp_recvfrom, |
778 | .xpo_sendto = svc_udp_sendto, |
779 | .xpo_result_payload = svc_sock_result_payload, |
780 | .xpo_release_ctxt = svc_udp_release_ctxt, |
781 | .xpo_detach = svc_sock_detach, |
782 | .xpo_free = svc_sock_free, |
783 | .xpo_has_wspace = svc_udp_has_wspace, |
784 | .xpo_accept = svc_udp_accept, |
785 | .xpo_kill_temp_xprt = svc_udp_kill_temp_xprt, |
786 | }; |
787 | |
788 | static struct svc_xprt_class svc_udp_class = { |
789 | .xcl_name = "udp" , |
790 | .xcl_owner = THIS_MODULE, |
791 | .xcl_ops = &svc_udp_ops, |
792 | .xcl_max_payload = RPCSVC_MAXPAYLOAD_UDP, |
793 | .xcl_ident = XPRT_TRANSPORT_UDP, |
794 | }; |
795 | |
796 | static void svc_udp_init(struct svc_sock *svsk, struct svc_serv *serv) |
797 | { |
798 | svc_xprt_init(sock_net(sk: svsk->sk_sock->sk), &svc_udp_class, |
799 | &svsk->sk_xprt, serv); |
800 | clear_bit(nr: XPT_CACHE_AUTH, addr: &svsk->sk_xprt.xpt_flags); |
801 | svsk->sk_sk->sk_data_ready = svc_data_ready; |
802 | svsk->sk_sk->sk_write_space = svc_write_space; |
803 | |
804 | /* initialise setting must have enough space to |
805 | * receive and respond to one request. |
806 | * svc_udp_recvfrom will re-adjust if necessary |
807 | */ |
808 | svc_sock_setbufsize(svsk, nreqs: 3); |
809 | |
810 | /* data might have come in before data_ready set up */ |
811 | set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags); |
812 | set_bit(nr: XPT_CHNGBUF, addr: &svsk->sk_xprt.xpt_flags); |
813 | |
814 | /* make sure we get destination address info */ |
815 | switch (svsk->sk_sk->sk_family) { |
816 | case AF_INET: |
817 | ip_sock_set_pktinfo(sk: svsk->sk_sock->sk); |
818 | break; |
819 | case AF_INET6: |
820 | ip6_sock_set_recvpktinfo(sk: svsk->sk_sock->sk); |
821 | break; |
822 | default: |
823 | BUG(); |
824 | } |
825 | } |
826 | |
827 | /* |
828 | * A data_ready event on a listening socket means there's a connection |
829 | * pending. Do not use state_change as a substitute for it. |
830 | */ |
831 | static void svc_tcp_listen_data_ready(struct sock *sk) |
832 | { |
833 | struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; |
834 | |
835 | trace_sk_data_ready(sk); |
836 | |
837 | /* |
838 | * This callback may called twice when a new connection |
839 | * is established as a child socket inherits everything |
840 | * from a parent LISTEN socket. |
841 | * 1) data_ready method of the parent socket will be called |
842 | * when one of child sockets become ESTABLISHED. |
843 | * 2) data_ready method of the child socket may be called |
844 | * when it receives data before the socket is accepted. |
845 | * In case of 2, we should ignore it silently and DO NOT |
846 | * dereference svsk. |
847 | */ |
848 | if (sk->sk_state != TCP_LISTEN) |
849 | return; |
850 | |
851 | if (svsk) { |
852 | /* Refer to svc_setup_socket() for details. */ |
853 | rmb(); |
854 | svsk->sk_odata(sk); |
855 | set_bit(nr: XPT_CONN, addr: &svsk->sk_xprt.xpt_flags); |
856 | svc_xprt_enqueue(xprt: &svsk->sk_xprt); |
857 | } |
858 | } |
859 | |
860 | /* |
861 | * A state change on a connected socket means it's dying or dead. |
862 | */ |
863 | static void svc_tcp_state_change(struct sock *sk) |
864 | { |
865 | struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; |
866 | |
867 | if (svsk) { |
868 | /* Refer to svc_setup_socket() for details. */ |
869 | rmb(); |
870 | svsk->sk_ostate(sk); |
871 | trace_svcsock_tcp_state(xprt: &svsk->sk_xprt, socket: svsk->sk_sock); |
872 | if (sk->sk_state != TCP_ESTABLISHED) |
873 | svc_xprt_deferred_close(xprt: &svsk->sk_xprt); |
874 | } |
875 | } |
876 | |
877 | /* |
878 | * Accept a TCP connection |
879 | */ |
880 | static struct svc_xprt *svc_tcp_accept(struct svc_xprt *xprt) |
881 | { |
882 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
883 | struct sockaddr_storage addr; |
884 | struct sockaddr *sin = (struct sockaddr *) &addr; |
885 | struct svc_serv *serv = svsk->sk_xprt.xpt_server; |
886 | struct socket *sock = svsk->sk_sock; |
887 | struct socket *newsock; |
888 | struct svc_sock *newsvsk; |
889 | int err, slen; |
890 | |
891 | if (!sock) |
892 | return NULL; |
893 | |
894 | clear_bit(nr: XPT_CONN, addr: &svsk->sk_xprt.xpt_flags); |
895 | err = kernel_accept(sock, newsock: &newsock, O_NONBLOCK); |
896 | if (err < 0) { |
897 | if (err != -EAGAIN) |
898 | trace_svcsock_accept_err(xprt, service: serv->sv_name, status: err); |
899 | return NULL; |
900 | } |
901 | if (IS_ERR(ptr: sock_alloc_file(sock: newsock, O_NONBLOCK, NULL))) |
902 | return NULL; |
903 | |
904 | set_bit(nr: XPT_CONN, addr: &svsk->sk_xprt.xpt_flags); |
905 | |
906 | err = kernel_getpeername(sock: newsock, addr: sin); |
907 | if (err < 0) { |
908 | trace_svcsock_getpeername_err(xprt, service: serv->sv_name, status: err); |
909 | goto failed; /* aborted connection or whatever */ |
910 | } |
911 | slen = err; |
912 | |
913 | /* Reset the inherited callbacks before calling svc_setup_socket */ |
914 | newsock->sk->sk_state_change = svsk->sk_ostate; |
915 | newsock->sk->sk_data_ready = svsk->sk_odata; |
916 | newsock->sk->sk_write_space = svsk->sk_owspace; |
917 | |
918 | /* make sure that a write doesn't block forever when |
919 | * low on memory |
920 | */ |
921 | newsock->sk->sk_sndtimeo = HZ*30; |
922 | |
923 | newsvsk = svc_setup_socket(serv, newsock, |
924 | flags: (SVC_SOCK_ANONYMOUS | SVC_SOCK_TEMPORARY)); |
925 | if (IS_ERR(ptr: newsvsk)) |
926 | goto failed; |
927 | svc_xprt_set_remote(xprt: &newsvsk->sk_xprt, sa: sin, salen: slen); |
928 | err = kernel_getsockname(sock: newsock, addr: sin); |
929 | slen = err; |
930 | if (unlikely(err < 0)) |
931 | slen = offsetof(struct sockaddr, sa_data); |
932 | svc_xprt_set_local(xprt: &newsvsk->sk_xprt, sa: sin, salen: slen); |
933 | |
934 | if (sock_is_loopback(sk: newsock->sk)) |
935 | set_bit(nr: XPT_LOCAL, addr: &newsvsk->sk_xprt.xpt_flags); |
936 | else |
937 | clear_bit(nr: XPT_LOCAL, addr: &newsvsk->sk_xprt.xpt_flags); |
938 | if (serv->sv_stats) |
939 | serv->sv_stats->nettcpconn++; |
940 | |
941 | return &newsvsk->sk_xprt; |
942 | |
943 | failed: |
944 | sockfd_put(newsock); |
945 | return NULL; |
946 | } |
947 | |
948 | static size_t svc_tcp_restore_pages(struct svc_sock *svsk, |
949 | struct svc_rqst *rqstp) |
950 | { |
951 | size_t len = svsk->sk_datalen; |
952 | unsigned int i, npages; |
953 | |
954 | if (!len) |
955 | return 0; |
956 | npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
957 | for (i = 0; i < npages; i++) { |
958 | if (rqstp->rq_pages[i] != NULL) |
959 | put_page(page: rqstp->rq_pages[i]); |
960 | BUG_ON(svsk->sk_pages[i] == NULL); |
961 | rqstp->rq_pages[i] = svsk->sk_pages[i]; |
962 | svsk->sk_pages[i] = NULL; |
963 | } |
964 | rqstp->rq_arg.head[0].iov_base = page_address(rqstp->rq_pages[0]); |
965 | return len; |
966 | } |
967 | |
968 | static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp) |
969 | { |
970 | unsigned int i, len, npages; |
971 | |
972 | if (svsk->sk_datalen == 0) |
973 | return; |
974 | len = svsk->sk_datalen; |
975 | npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
976 | for (i = 0; i < npages; i++) { |
977 | svsk->sk_pages[i] = rqstp->rq_pages[i]; |
978 | rqstp->rq_pages[i] = NULL; |
979 | } |
980 | } |
981 | |
982 | static void svc_tcp_clear_pages(struct svc_sock *svsk) |
983 | { |
984 | unsigned int i, len, npages; |
985 | |
986 | if (svsk->sk_datalen == 0) |
987 | goto out; |
988 | len = svsk->sk_datalen; |
989 | npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
990 | for (i = 0; i < npages; i++) { |
991 | if (svsk->sk_pages[i] == NULL) { |
992 | WARN_ON_ONCE(1); |
993 | continue; |
994 | } |
995 | put_page(page: svsk->sk_pages[i]); |
996 | svsk->sk_pages[i] = NULL; |
997 | } |
998 | out: |
999 | svsk->sk_tcplen = 0; |
1000 | svsk->sk_datalen = 0; |
1001 | } |
1002 | |
1003 | /* |
1004 | * Receive fragment record header into sk_marker. |
1005 | */ |
1006 | static ssize_t svc_tcp_read_marker(struct svc_sock *svsk, |
1007 | struct svc_rqst *rqstp) |
1008 | { |
1009 | ssize_t want, len; |
1010 | |
1011 | /* If we haven't gotten the record length yet, |
1012 | * get the next four bytes. |
1013 | */ |
1014 | if (svsk->sk_tcplen < sizeof(rpc_fraghdr)) { |
1015 | struct msghdr msg = { NULL }; |
1016 | struct kvec iov; |
1017 | |
1018 | want = sizeof(rpc_fraghdr) - svsk->sk_tcplen; |
1019 | iov.iov_base = ((char *)&svsk->sk_marker) + svsk->sk_tcplen; |
1020 | iov.iov_len = want; |
1021 | iov_iter_kvec(i: &msg.msg_iter, ITER_DEST, kvec: &iov, nr_segs: 1, count: want); |
1022 | len = svc_tcp_sock_recv_cmsg(svsk, msg: &msg); |
1023 | if (len < 0) |
1024 | return len; |
1025 | svsk->sk_tcplen += len; |
1026 | if (len < want) { |
1027 | /* call again to read the remaining bytes */ |
1028 | goto err_short; |
1029 | } |
1030 | trace_svcsock_marker(xprt: &svsk->sk_xprt, marker: svsk->sk_marker); |
1031 | if (svc_sock_reclen(svsk) + svsk->sk_datalen > |
1032 | svsk->sk_xprt.xpt_server->sv_max_mesg) |
1033 | goto err_too_large; |
1034 | } |
1035 | return svc_sock_reclen(svsk); |
1036 | |
1037 | err_too_large: |
1038 | net_notice_ratelimited("svc: %s %s RPC fragment too large: %d\n" , |
1039 | __func__, svsk->sk_xprt.xpt_server->sv_name, |
1040 | svc_sock_reclen(svsk)); |
1041 | svc_xprt_deferred_close(xprt: &svsk->sk_xprt); |
1042 | err_short: |
1043 | return -EAGAIN; |
1044 | } |
1045 | |
1046 | static int receive_cb_reply(struct svc_sock *svsk, struct svc_rqst *rqstp) |
1047 | { |
1048 | struct rpc_xprt *bc_xprt = svsk->sk_xprt.xpt_bc_xprt; |
1049 | struct rpc_rqst *req = NULL; |
1050 | struct kvec *src, *dst; |
1051 | __be32 *p = (__be32 *)rqstp->rq_arg.head[0].iov_base; |
1052 | __be32 xid; |
1053 | __be32 calldir; |
1054 | |
1055 | xid = *p++; |
1056 | calldir = *p; |
1057 | |
1058 | if (!bc_xprt) |
1059 | return -EAGAIN; |
1060 | spin_lock(lock: &bc_xprt->queue_lock); |
1061 | req = xprt_lookup_rqst(xprt: bc_xprt, xid); |
1062 | if (!req) |
1063 | goto unlock_notfound; |
1064 | |
1065 | memcpy(&req->rq_private_buf, &req->rq_rcv_buf, sizeof(struct xdr_buf)); |
1066 | /* |
1067 | * XXX!: cheating for now! Only copying HEAD. |
1068 | * But we know this is good enough for now (in fact, for any |
1069 | * callback reply in the forseeable future). |
1070 | */ |
1071 | dst = &req->rq_private_buf.head[0]; |
1072 | src = &rqstp->rq_arg.head[0]; |
1073 | if (dst->iov_len < src->iov_len) |
1074 | goto unlock_eagain; /* whatever; just giving up. */ |
1075 | memcpy(dst->iov_base, src->iov_base, src->iov_len); |
1076 | xprt_complete_rqst(task: req->rq_task, copied: rqstp->rq_arg.len); |
1077 | rqstp->rq_arg.len = 0; |
1078 | spin_unlock(lock: &bc_xprt->queue_lock); |
1079 | return 0; |
1080 | unlock_notfound: |
1081 | printk(KERN_NOTICE |
1082 | "%s: Got unrecognized reply: " |
1083 | "calldir 0x%x xpt_bc_xprt %p xid %08x\n" , |
1084 | __func__, ntohl(calldir), |
1085 | bc_xprt, ntohl(xid)); |
1086 | unlock_eagain: |
1087 | spin_unlock(lock: &bc_xprt->queue_lock); |
1088 | return -EAGAIN; |
1089 | } |
1090 | |
1091 | static void svc_tcp_fragment_received(struct svc_sock *svsk) |
1092 | { |
1093 | /* If we have more data, signal svc_xprt_enqueue() to try again */ |
1094 | svsk->sk_tcplen = 0; |
1095 | svsk->sk_marker = xdr_zero; |
1096 | |
1097 | smp_wmb(); |
1098 | tcp_set_rcvlowat(sk: svsk->sk_sk, val: 1); |
1099 | } |
1100 | |
1101 | /** |
1102 | * svc_tcp_recvfrom - Receive data from a TCP socket |
1103 | * @rqstp: request structure into which to receive an RPC Call |
1104 | * |
1105 | * Called in a loop when XPT_DATA has been set. |
1106 | * |
1107 | * Read the 4-byte stream record marker, then use the record length |
1108 | * in that marker to set up exactly the resources needed to receive |
1109 | * the next RPC message into @rqstp. |
1110 | * |
1111 | * Returns: |
1112 | * On success, the number of bytes in a received RPC Call, or |
1113 | * %0 if a complete RPC Call message was not ready to return |
1114 | * |
1115 | * The zero return case handles partial receives and callback Replies. |
1116 | * The state of a partial receive is preserved in the svc_sock for |
1117 | * the next call to svc_tcp_recvfrom. |
1118 | */ |
1119 | static int svc_tcp_recvfrom(struct svc_rqst *rqstp) |
1120 | { |
1121 | struct svc_sock *svsk = |
1122 | container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); |
1123 | struct svc_serv *serv = svsk->sk_xprt.xpt_server; |
1124 | size_t want, base; |
1125 | ssize_t len; |
1126 | __be32 *p; |
1127 | __be32 calldir; |
1128 | |
1129 | clear_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags); |
1130 | len = svc_tcp_read_marker(svsk, rqstp); |
1131 | if (len < 0) |
1132 | goto error; |
1133 | |
1134 | base = svc_tcp_restore_pages(svsk, rqstp); |
1135 | want = len - (svsk->sk_tcplen - sizeof(rpc_fraghdr)); |
1136 | len = svc_tcp_read_msg(rqstp, buflen: base + want, seek: base); |
1137 | if (len >= 0) { |
1138 | trace_svcsock_tcp_recv(xprt: &svsk->sk_xprt, result: len); |
1139 | svsk->sk_tcplen += len; |
1140 | svsk->sk_datalen += len; |
1141 | } |
1142 | if (len != want || !svc_sock_final_rec(svsk)) |
1143 | goto err_incomplete; |
1144 | if (svsk->sk_datalen < 8) |
1145 | goto err_nuts; |
1146 | |
1147 | rqstp->rq_arg.len = svsk->sk_datalen; |
1148 | rqstp->rq_arg.page_base = 0; |
1149 | if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) { |
1150 | rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len; |
1151 | rqstp->rq_arg.page_len = 0; |
1152 | } else |
1153 | rqstp->rq_arg.page_len = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len; |
1154 | |
1155 | rqstp->rq_xprt_ctxt = NULL; |
1156 | rqstp->rq_prot = IPPROTO_TCP; |
1157 | if (test_bit(XPT_LOCAL, &svsk->sk_xprt.xpt_flags)) |
1158 | set_bit(nr: RQ_LOCAL, addr: &rqstp->rq_flags); |
1159 | else |
1160 | clear_bit(nr: RQ_LOCAL, addr: &rqstp->rq_flags); |
1161 | |
1162 | p = (__be32 *)rqstp->rq_arg.head[0].iov_base; |
1163 | calldir = p[1]; |
1164 | if (calldir) |
1165 | len = receive_cb_reply(svsk, rqstp); |
1166 | |
1167 | /* Reset TCP read info */ |
1168 | svsk->sk_datalen = 0; |
1169 | svc_tcp_fragment_received(svsk); |
1170 | |
1171 | if (len < 0) |
1172 | goto error; |
1173 | |
1174 | svc_xprt_copy_addrs(rqstp, xprt: &svsk->sk_xprt); |
1175 | if (serv->sv_stats) |
1176 | serv->sv_stats->nettcpcnt++; |
1177 | |
1178 | svc_sock_secure_port(rqstp); |
1179 | svc_xprt_received(xprt: rqstp->rq_xprt); |
1180 | return rqstp->rq_arg.len; |
1181 | |
1182 | err_incomplete: |
1183 | svc_tcp_save_pages(svsk, rqstp); |
1184 | if (len < 0 && len != -EAGAIN) |
1185 | goto err_delete; |
1186 | if (len == want) |
1187 | svc_tcp_fragment_received(svsk); |
1188 | else { |
1189 | /* Avoid more ->sk_data_ready() calls until the rest |
1190 | * of the message has arrived. This reduces service |
1191 | * thread wake-ups on large incoming messages. */ |
1192 | tcp_set_rcvlowat(sk: svsk->sk_sk, |
1193 | val: svc_sock_reclen(svsk) - svsk->sk_tcplen); |
1194 | |
1195 | trace_svcsock_tcp_recv_short(xprt: &svsk->sk_xprt, |
1196 | expected: svc_sock_reclen(svsk), |
1197 | received: svsk->sk_tcplen - sizeof(rpc_fraghdr)); |
1198 | } |
1199 | goto err_noclose; |
1200 | error: |
1201 | if (len != -EAGAIN) |
1202 | goto err_delete; |
1203 | trace_svcsock_tcp_recv_eagain(xprt: &svsk->sk_xprt, result: 0); |
1204 | goto err_noclose; |
1205 | err_nuts: |
1206 | svsk->sk_datalen = 0; |
1207 | err_delete: |
1208 | trace_svcsock_tcp_recv_err(xprt: &svsk->sk_xprt, result: len); |
1209 | svc_xprt_deferred_close(xprt: &svsk->sk_xprt); |
1210 | err_noclose: |
1211 | svc_xprt_received(xprt: rqstp->rq_xprt); |
1212 | return 0; /* record not complete */ |
1213 | } |
1214 | |
1215 | /* |
1216 | * MSG_SPLICE_PAGES is used exclusively to reduce the number of |
1217 | * copy operations in this path. Therefore the caller must ensure |
1218 | * that the pages backing @xdr are unchanging. |
1219 | * |
1220 | * Note that the send is non-blocking. The caller has incremented |
1221 | * the reference count on each page backing the RPC message, and |
1222 | * the network layer will "put" these pages when transmission is |
1223 | * complete. |
1224 | * |
1225 | * This is safe for our RPC services because the memory backing |
1226 | * the head and tail components is never kmalloc'd. These always |
1227 | * come from pages in the svc_rqst::rq_pages array. |
1228 | */ |
1229 | static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp, |
1230 | rpc_fraghdr marker, unsigned int *sentp) |
1231 | { |
1232 | struct msghdr msg = { |
1233 | .msg_flags = MSG_SPLICE_PAGES, |
1234 | }; |
1235 | unsigned int count; |
1236 | void *buf; |
1237 | int ret; |
1238 | |
1239 | *sentp = 0; |
1240 | |
1241 | /* The stream record marker is copied into a temporary page |
1242 | * fragment buffer so that it can be included in rq_bvec. |
1243 | */ |
1244 | buf = page_frag_alloc(nc: &svsk->sk_frag_cache, fragsz: sizeof(marker), |
1245 | GFP_KERNEL); |
1246 | if (!buf) |
1247 | return -ENOMEM; |
1248 | memcpy(buf, &marker, sizeof(marker)); |
1249 | bvec_set_virt(bv: rqstp->rq_bvec, vaddr: buf, len: sizeof(marker)); |
1250 | |
1251 | count = xdr_buf_to_bvec(bvec: rqstp->rq_bvec + 1, |
1252 | ARRAY_SIZE(rqstp->rq_bvec) - 1, xdr: &rqstp->rq_res); |
1253 | |
1254 | iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: rqstp->rq_bvec, |
1255 | nr_segs: 1 + count, count: sizeof(marker) + rqstp->rq_res.len); |
1256 | ret = sock_sendmsg(sock: svsk->sk_sock, msg: &msg); |
1257 | if (ret < 0) |
1258 | return ret; |
1259 | *sentp += ret; |
1260 | return 0; |
1261 | } |
1262 | |
1263 | /** |
1264 | * svc_tcp_sendto - Send out a reply on a TCP socket |
1265 | * @rqstp: completed svc_rqst |
1266 | * |
1267 | * xpt_mutex ensures @rqstp's whole message is written to the socket |
1268 | * without interruption. |
1269 | * |
1270 | * Returns the number of bytes sent, or a negative errno. |
1271 | */ |
1272 | static int svc_tcp_sendto(struct svc_rqst *rqstp) |
1273 | { |
1274 | struct svc_xprt *xprt = rqstp->rq_xprt; |
1275 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
1276 | struct xdr_buf *xdr = &rqstp->rq_res; |
1277 | rpc_fraghdr marker = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | |
1278 | (u32)xdr->len); |
1279 | unsigned int sent; |
1280 | int err; |
1281 | |
1282 | svc_tcp_release_ctxt(xprt, ctxt: rqstp->rq_xprt_ctxt); |
1283 | rqstp->rq_xprt_ctxt = NULL; |
1284 | |
1285 | mutex_lock(&xprt->xpt_mutex); |
1286 | if (svc_xprt_is_dead(xprt)) |
1287 | goto out_notconn; |
1288 | err = svc_tcp_sendmsg(svsk, rqstp, marker, sentp: &sent); |
1289 | trace_svcsock_tcp_send(xprt, result: err < 0 ? (long)err : sent); |
1290 | if (err < 0 || sent != (xdr->len + sizeof(marker))) |
1291 | goto out_close; |
1292 | mutex_unlock(lock: &xprt->xpt_mutex); |
1293 | return sent; |
1294 | |
1295 | out_notconn: |
1296 | mutex_unlock(lock: &xprt->xpt_mutex); |
1297 | return -ENOTCONN; |
1298 | out_close: |
1299 | pr_notice("rpc-srv/tcp: %s: %s %d when sending %d bytes - shutting down socket\n" , |
1300 | xprt->xpt_server->sv_name, |
1301 | (err < 0) ? "got error" : "sent" , |
1302 | (err < 0) ? err : sent, xdr->len); |
1303 | svc_xprt_deferred_close(xprt); |
1304 | mutex_unlock(lock: &xprt->xpt_mutex); |
1305 | return -EAGAIN; |
1306 | } |
1307 | |
1308 | static struct svc_xprt *svc_tcp_create(struct svc_serv *serv, |
1309 | struct net *net, |
1310 | struct sockaddr *sa, int salen, |
1311 | int flags) |
1312 | { |
1313 | return svc_create_socket(serv, IPPROTO_TCP, net, sa, salen, flags); |
1314 | } |
1315 | |
1316 | static const struct svc_xprt_ops svc_tcp_ops = { |
1317 | .xpo_create = svc_tcp_create, |
1318 | .xpo_recvfrom = svc_tcp_recvfrom, |
1319 | .xpo_sendto = svc_tcp_sendto, |
1320 | .xpo_result_payload = svc_sock_result_payload, |
1321 | .xpo_release_ctxt = svc_tcp_release_ctxt, |
1322 | .xpo_detach = svc_tcp_sock_detach, |
1323 | .xpo_free = svc_sock_free, |
1324 | .xpo_has_wspace = svc_tcp_has_wspace, |
1325 | .xpo_accept = svc_tcp_accept, |
1326 | .xpo_kill_temp_xprt = svc_tcp_kill_temp_xprt, |
1327 | .xpo_handshake = svc_tcp_handshake, |
1328 | }; |
1329 | |
1330 | static struct svc_xprt_class svc_tcp_class = { |
1331 | .xcl_name = "tcp" , |
1332 | .xcl_owner = THIS_MODULE, |
1333 | .xcl_ops = &svc_tcp_ops, |
1334 | .xcl_max_payload = RPCSVC_MAXPAYLOAD_TCP, |
1335 | .xcl_ident = XPRT_TRANSPORT_TCP, |
1336 | }; |
1337 | |
1338 | void svc_init_xprt_sock(void) |
1339 | { |
1340 | svc_reg_xprt_class(&svc_tcp_class); |
1341 | svc_reg_xprt_class(&svc_udp_class); |
1342 | } |
1343 | |
1344 | void svc_cleanup_xprt_sock(void) |
1345 | { |
1346 | svc_unreg_xprt_class(&svc_tcp_class); |
1347 | svc_unreg_xprt_class(&svc_udp_class); |
1348 | } |
1349 | |
1350 | static void svc_tcp_init(struct svc_sock *svsk, struct svc_serv *serv) |
1351 | { |
1352 | struct sock *sk = svsk->sk_sk; |
1353 | |
1354 | svc_xprt_init(sock_net(sk: svsk->sk_sock->sk), &svc_tcp_class, |
1355 | &svsk->sk_xprt, serv); |
1356 | set_bit(nr: XPT_CACHE_AUTH, addr: &svsk->sk_xprt.xpt_flags); |
1357 | set_bit(nr: XPT_CONG_CTRL, addr: &svsk->sk_xprt.xpt_flags); |
1358 | if (sk->sk_state == TCP_LISTEN) { |
1359 | strcpy(p: svsk->sk_xprt.xpt_remotebuf, q: "listener" ); |
1360 | set_bit(nr: XPT_LISTENER, addr: &svsk->sk_xprt.xpt_flags); |
1361 | sk->sk_data_ready = svc_tcp_listen_data_ready; |
1362 | set_bit(nr: XPT_CONN, addr: &svsk->sk_xprt.xpt_flags); |
1363 | } else { |
1364 | sk->sk_state_change = svc_tcp_state_change; |
1365 | sk->sk_data_ready = svc_data_ready; |
1366 | sk->sk_write_space = svc_write_space; |
1367 | |
1368 | svsk->sk_marker = xdr_zero; |
1369 | svsk->sk_tcplen = 0; |
1370 | svsk->sk_datalen = 0; |
1371 | memset(&svsk->sk_pages[0], 0, sizeof(svsk->sk_pages)); |
1372 | |
1373 | tcp_sock_set_nodelay(sk); |
1374 | |
1375 | set_bit(nr: XPT_DATA, addr: &svsk->sk_xprt.xpt_flags); |
1376 | switch (sk->sk_state) { |
1377 | case TCP_SYN_RECV: |
1378 | case TCP_ESTABLISHED: |
1379 | break; |
1380 | default: |
1381 | svc_xprt_deferred_close(xprt: &svsk->sk_xprt); |
1382 | } |
1383 | } |
1384 | } |
1385 | |
1386 | void svc_sock_update_bufs(struct svc_serv *serv) |
1387 | { |
1388 | /* |
1389 | * The number of server threads has changed. Update |
1390 | * rcvbuf and sndbuf accordingly on all sockets |
1391 | */ |
1392 | struct svc_sock *svsk; |
1393 | |
1394 | spin_lock_bh(lock: &serv->sv_lock); |
1395 | list_for_each_entry(svsk, &serv->sv_permsocks, sk_xprt.xpt_list) |
1396 | set_bit(nr: XPT_CHNGBUF, addr: &svsk->sk_xprt.xpt_flags); |
1397 | spin_unlock_bh(lock: &serv->sv_lock); |
1398 | } |
1399 | EXPORT_SYMBOL_GPL(svc_sock_update_bufs); |
1400 | |
1401 | /* |
1402 | * Initialize socket for RPC use and create svc_sock struct |
1403 | */ |
1404 | static struct svc_sock *svc_setup_socket(struct svc_serv *serv, |
1405 | struct socket *sock, |
1406 | int flags) |
1407 | { |
1408 | struct svc_sock *svsk; |
1409 | struct sock *inet; |
1410 | int pmap_register = !(flags & SVC_SOCK_ANONYMOUS); |
1411 | |
1412 | svsk = kzalloc(size: sizeof(*svsk), GFP_KERNEL); |
1413 | if (!svsk) |
1414 | return ERR_PTR(error: -ENOMEM); |
1415 | |
1416 | inet = sock->sk; |
1417 | |
1418 | if (pmap_register) { |
1419 | int err; |
1420 | |
1421 | err = svc_register(serv, sock_net(sk: sock->sk), inet->sk_family, |
1422 | inet->sk_protocol, |
1423 | ntohs(inet_sk(inet)->inet_sport)); |
1424 | if (err < 0) { |
1425 | kfree(objp: svsk); |
1426 | return ERR_PTR(error: err); |
1427 | } |
1428 | } |
1429 | |
1430 | svsk->sk_sock = sock; |
1431 | svsk->sk_sk = inet; |
1432 | svsk->sk_ostate = inet->sk_state_change; |
1433 | svsk->sk_odata = inet->sk_data_ready; |
1434 | svsk->sk_owspace = inet->sk_write_space; |
1435 | /* |
1436 | * This barrier is necessary in order to prevent race condition |
1437 | * with svc_data_ready(), svc_tcp_listen_data_ready(), and others |
1438 | * when calling callbacks above. |
1439 | */ |
1440 | wmb(); |
1441 | inet->sk_user_data = svsk; |
1442 | |
1443 | /* Initialize the socket */ |
1444 | if (sock->type == SOCK_DGRAM) |
1445 | svc_udp_init(svsk, serv); |
1446 | else |
1447 | svc_tcp_init(svsk, serv); |
1448 | |
1449 | trace_svcsock_new(svsk, socket: sock); |
1450 | return svsk; |
1451 | } |
1452 | |
1453 | /** |
1454 | * svc_addsock - add a listener socket to an RPC service |
1455 | * @serv: pointer to RPC service to which to add a new listener |
1456 | * @net: caller's network namespace |
1457 | * @fd: file descriptor of the new listener |
1458 | * @name_return: pointer to buffer to fill in with name of listener |
1459 | * @len: size of the buffer |
1460 | * @cred: credential |
1461 | * |
1462 | * Fills in socket name and returns positive length of name if successful. |
1463 | * Name is terminated with '\n'. On error, returns a negative errno |
1464 | * value. |
1465 | */ |
1466 | int svc_addsock(struct svc_serv *serv, struct net *net, const int fd, |
1467 | char *name_return, const size_t len, const struct cred *cred) |
1468 | { |
1469 | int err = 0; |
1470 | struct socket *so = sockfd_lookup(fd, err: &err); |
1471 | struct svc_sock *svsk = NULL; |
1472 | struct sockaddr_storage addr; |
1473 | struct sockaddr *sin = (struct sockaddr *)&addr; |
1474 | int salen; |
1475 | |
1476 | if (!so) |
1477 | return err; |
1478 | err = -EINVAL; |
1479 | if (sock_net(sk: so->sk) != net) |
1480 | goto out; |
1481 | err = -EAFNOSUPPORT; |
1482 | if ((so->sk->sk_family != PF_INET) && (so->sk->sk_family != PF_INET6)) |
1483 | goto out; |
1484 | err = -EPROTONOSUPPORT; |
1485 | if (so->sk->sk_protocol != IPPROTO_TCP && |
1486 | so->sk->sk_protocol != IPPROTO_UDP) |
1487 | goto out; |
1488 | err = -EISCONN; |
1489 | if (so->state > SS_UNCONNECTED) |
1490 | goto out; |
1491 | err = -ENOENT; |
1492 | if (!try_module_get(THIS_MODULE)) |
1493 | goto out; |
1494 | svsk = svc_setup_socket(serv, sock: so, SVC_SOCK_DEFAULTS); |
1495 | if (IS_ERR(ptr: svsk)) { |
1496 | module_put(THIS_MODULE); |
1497 | err = PTR_ERR(ptr: svsk); |
1498 | goto out; |
1499 | } |
1500 | salen = kernel_getsockname(sock: svsk->sk_sock, addr: sin); |
1501 | if (salen >= 0) |
1502 | svc_xprt_set_local(xprt: &svsk->sk_xprt, sa: sin, salen); |
1503 | svsk->sk_xprt.xpt_cred = get_cred(cred); |
1504 | svc_add_new_perm_xprt(serv, xprt: &svsk->sk_xprt); |
1505 | return svc_one_sock_name(svsk, buf: name_return, remaining: len); |
1506 | out: |
1507 | sockfd_put(so); |
1508 | return err; |
1509 | } |
1510 | EXPORT_SYMBOL_GPL(svc_addsock); |
1511 | |
1512 | /* |
1513 | * Create socket for RPC service. |
1514 | */ |
1515 | static struct svc_xprt *svc_create_socket(struct svc_serv *serv, |
1516 | int protocol, |
1517 | struct net *net, |
1518 | struct sockaddr *sin, int len, |
1519 | int flags) |
1520 | { |
1521 | struct svc_sock *svsk; |
1522 | struct socket *sock; |
1523 | int error; |
1524 | int type; |
1525 | struct sockaddr_storage addr; |
1526 | struct sockaddr *newsin = (struct sockaddr *)&addr; |
1527 | int newlen; |
1528 | int family; |
1529 | |
1530 | if (protocol != IPPROTO_UDP && protocol != IPPROTO_TCP) { |
1531 | printk(KERN_WARNING "svc: only UDP and TCP " |
1532 | "sockets supported\n" ); |
1533 | return ERR_PTR(error: -EINVAL); |
1534 | } |
1535 | |
1536 | type = (protocol == IPPROTO_UDP)? SOCK_DGRAM : SOCK_STREAM; |
1537 | switch (sin->sa_family) { |
1538 | case AF_INET6: |
1539 | family = PF_INET6; |
1540 | break; |
1541 | case AF_INET: |
1542 | family = PF_INET; |
1543 | break; |
1544 | default: |
1545 | return ERR_PTR(error: -EINVAL); |
1546 | } |
1547 | |
1548 | error = __sock_create(net, family, type, proto: protocol, res: &sock, kern: 1); |
1549 | if (error < 0) |
1550 | return ERR_PTR(error); |
1551 | |
1552 | svc_reclassify_socket(sock); |
1553 | |
1554 | /* |
1555 | * If this is an PF_INET6 listener, we want to avoid |
1556 | * getting requests from IPv4 remotes. Those should |
1557 | * be shunted to a PF_INET listener via rpcbind. |
1558 | */ |
1559 | if (family == PF_INET6) |
1560 | ip6_sock_set_v6only(sk: sock->sk); |
1561 | if (type == SOCK_STREAM) |
1562 | sock->sk->sk_reuse = SK_CAN_REUSE; /* allow address reuse */ |
1563 | error = kernel_bind(sock, addr: sin, addrlen: len); |
1564 | if (error < 0) |
1565 | goto bummer; |
1566 | |
1567 | error = kernel_getsockname(sock, addr: newsin); |
1568 | if (error < 0) |
1569 | goto bummer; |
1570 | newlen = error; |
1571 | |
1572 | if (protocol == IPPROTO_TCP) { |
1573 | if ((error = kernel_listen(sock, backlog: 64)) < 0) |
1574 | goto bummer; |
1575 | } |
1576 | |
1577 | svsk = svc_setup_socket(serv, sock, flags); |
1578 | if (IS_ERR(ptr: svsk)) { |
1579 | error = PTR_ERR(ptr: svsk); |
1580 | goto bummer; |
1581 | } |
1582 | svc_xprt_set_local(xprt: &svsk->sk_xprt, sa: newsin, salen: newlen); |
1583 | return (struct svc_xprt *)svsk; |
1584 | bummer: |
1585 | sock_release(sock); |
1586 | return ERR_PTR(error); |
1587 | } |
1588 | |
1589 | /* |
1590 | * Detach the svc_sock from the socket so that no |
1591 | * more callbacks occur. |
1592 | */ |
1593 | static void svc_sock_detach(struct svc_xprt *xprt) |
1594 | { |
1595 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
1596 | struct sock *sk = svsk->sk_sk; |
1597 | |
1598 | /* put back the old socket callbacks */ |
1599 | lock_sock(sk); |
1600 | sk->sk_state_change = svsk->sk_ostate; |
1601 | sk->sk_data_ready = svsk->sk_odata; |
1602 | sk->sk_write_space = svsk->sk_owspace; |
1603 | sk->sk_user_data = NULL; |
1604 | release_sock(sk); |
1605 | } |
1606 | |
1607 | /* |
1608 | * Disconnect the socket, and reset the callbacks |
1609 | */ |
1610 | static void svc_tcp_sock_detach(struct svc_xprt *xprt) |
1611 | { |
1612 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
1613 | |
1614 | tls_handshake_close(sock: svsk->sk_sock); |
1615 | |
1616 | svc_sock_detach(xprt); |
1617 | |
1618 | if (!test_bit(XPT_LISTENER, &xprt->xpt_flags)) { |
1619 | svc_tcp_clear_pages(svsk); |
1620 | kernel_sock_shutdown(sock: svsk->sk_sock, how: SHUT_RDWR); |
1621 | } |
1622 | } |
1623 | |
1624 | /* |
1625 | * Free the svc_sock's socket resources and the svc_sock itself. |
1626 | */ |
1627 | static void svc_sock_free(struct svc_xprt *xprt) |
1628 | { |
1629 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
1630 | struct page_frag_cache *pfc = &svsk->sk_frag_cache; |
1631 | struct socket *sock = svsk->sk_sock; |
1632 | |
1633 | trace_svcsock_free(svsk, socket: sock); |
1634 | |
1635 | tls_handshake_cancel(sk: sock->sk); |
1636 | if (sock->file) |
1637 | sockfd_put(sock); |
1638 | else |
1639 | sock_release(sock); |
1640 | if (pfc->va) |
1641 | __page_frag_cache_drain(page: virt_to_head_page(x: pfc->va), |
1642 | count: pfc->pagecnt_bias); |
1643 | kfree(objp: svsk); |
1644 | } |
1645 | |