1 | /* |
2 | * Copyright (c) 2006, 2017 Oracle and/or its affiliates. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | * |
32 | */ |
33 | #include <linux/kernel.h> |
34 | #include <linux/in.h> |
35 | #include <net/tcp.h> |
36 | |
37 | #include "rds_single_path.h" |
38 | #include "rds.h" |
39 | #include "tcp.h" |
40 | |
41 | void rds_tcp_xmit_path_prepare(struct rds_conn_path *cp) |
42 | { |
43 | struct rds_tcp_connection *tc = cp->cp_transport_data; |
44 | |
45 | tcp_sock_set_cork(sk: tc->t_sock->sk, on: true); |
46 | } |
47 | |
48 | void rds_tcp_xmit_path_complete(struct rds_conn_path *cp) |
49 | { |
50 | struct rds_tcp_connection *tc = cp->cp_transport_data; |
51 | |
52 | tcp_sock_set_cork(sk: tc->t_sock->sk, on: false); |
53 | } |
54 | |
55 | /* the core send_sem serializes this with other xmit and shutdown */ |
56 | static int rds_tcp_sendmsg(struct socket *sock, void *data, unsigned int len) |
57 | { |
58 | struct kvec vec = { |
59 | .iov_base = data, |
60 | .iov_len = len, |
61 | }; |
62 | struct msghdr msg = { |
63 | .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL, |
64 | }; |
65 | |
66 | return kernel_sendmsg(sock, msg: &msg, vec: &vec, num: 1, len: vec.iov_len); |
67 | } |
68 | |
69 | /* the core send_sem serializes this with other xmit and shutdown */ |
70 | int rds_tcp_xmit(struct rds_connection *conn, struct rds_message *rm, |
71 | unsigned int hdr_off, unsigned int sg, unsigned int off) |
72 | { |
73 | struct rds_conn_path *cp = rm->m_inc.i_conn_path; |
74 | struct rds_tcp_connection *tc = cp->cp_transport_data; |
75 | struct msghdr msg = {}; |
76 | struct bio_vec bvec; |
77 | int done = 0; |
78 | int ret = 0; |
79 | |
80 | if (hdr_off == 0) { |
81 | /* |
82 | * m_ack_seq is set to the sequence number of the last byte of |
83 | * header and data. see rds_tcp_is_acked(). |
84 | */ |
85 | tc->t_last_sent_nxt = rds_tcp_write_seq(tc); |
86 | rm->m_ack_seq = tc->t_last_sent_nxt + |
87 | sizeof(struct rds_header) + |
88 | be32_to_cpu(rm->m_inc.i_hdr.h_len) - 1; |
89 | smp_mb__before_atomic(); |
90 | set_bit(RDS_MSG_HAS_ACK_SEQ, addr: &rm->m_flags); |
91 | tc->t_last_expected_una = rm->m_ack_seq + 1; |
92 | |
93 | if (test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags)) |
94 | rm->m_inc.i_hdr.h_flags |= RDS_FLAG_RETRANSMITTED; |
95 | |
96 | rdsdebug("rm %p tcp nxt %u ack_seq %llu\n" , |
97 | rm, rds_tcp_write_seq(tc), |
98 | (unsigned long long)rm->m_ack_seq); |
99 | } |
100 | |
101 | if (hdr_off < sizeof(struct rds_header)) { |
102 | /* see rds_tcp_write_space() */ |
103 | set_bit(SOCK_NOSPACE, addr: &tc->t_sock->sk->sk_socket->flags); |
104 | |
105 | ret = rds_tcp_sendmsg(sock: tc->t_sock, |
106 | data: (void *)&rm->m_inc.i_hdr + hdr_off, |
107 | len: sizeof(rm->m_inc.i_hdr) - hdr_off); |
108 | if (ret < 0) |
109 | goto out; |
110 | done += ret; |
111 | if (hdr_off + done != sizeof(struct rds_header)) |
112 | goto out; |
113 | } |
114 | |
115 | while (sg < rm->data.op_nents) { |
116 | msg.msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT | MSG_NOSIGNAL; |
117 | if (sg + 1 < rm->data.op_nents) |
118 | msg.msg_flags |= MSG_MORE; |
119 | |
120 | bvec_set_page(bv: &bvec, page: sg_page(sg: &rm->data.op_sg[sg]), |
121 | len: rm->data.op_sg[sg].length - off, |
122 | offset: rm->data.op_sg[sg].offset + off); |
123 | iov_iter_bvec(i: &msg.msg_iter, ITER_SOURCE, bvec: &bvec, nr_segs: 1, |
124 | count: rm->data.op_sg[sg].length - off); |
125 | ret = sock_sendmsg(sock: tc->t_sock, msg: &msg); |
126 | rdsdebug("tcp sendpage %p:%u:%u ret %d\n" , (void *)sg_page(&rm->data.op_sg[sg]), |
127 | rm->data.op_sg[sg].offset + off, rm->data.op_sg[sg].length - off, |
128 | ret); |
129 | if (ret <= 0) |
130 | break; |
131 | |
132 | off += ret; |
133 | done += ret; |
134 | if (off == rm->data.op_sg[sg].length) { |
135 | off = 0; |
136 | sg++; |
137 | } |
138 | } |
139 | |
140 | out: |
141 | if (ret <= 0) { |
142 | /* write_space will hit after EAGAIN, all else fatal */ |
143 | if (ret == -EAGAIN) { |
144 | rds_tcp_stats_inc(s_tcp_sndbuf_full); |
145 | ret = 0; |
146 | } else { |
147 | /* No need to disconnect/reconnect if path_drop |
148 | * has already been triggered, because, e.g., of |
149 | * an incoming RST. |
150 | */ |
151 | if (rds_conn_path_up(cp)) { |
152 | pr_warn("RDS/tcp: send to %pI6c on cp [%d]" |
153 | "returned %d, " |
154 | "disconnecting and reconnecting\n" , |
155 | &conn->c_faddr, cp->cp_index, ret); |
156 | rds_conn_path_drop(cpath: cp, destroy: false); |
157 | } |
158 | } |
159 | } |
160 | if (done == 0) |
161 | done = ret; |
162 | return done; |
163 | } |
164 | |
165 | /* |
166 | * rm->m_ack_seq is set to the tcp sequence number that corresponds to the |
167 | * last byte of the message, including the header. This means that the |
168 | * entire message has been received if rm->m_ack_seq is "before" the next |
169 | * unacked byte of the TCP sequence space. We have to do very careful |
170 | * wrapping 32bit comparisons here. |
171 | */ |
172 | static int rds_tcp_is_acked(struct rds_message *rm, uint64_t ack) |
173 | { |
174 | if (!test_bit(RDS_MSG_HAS_ACK_SEQ, &rm->m_flags)) |
175 | return 0; |
176 | return (__s32)((u32)rm->m_ack_seq - (u32)ack) < 0; |
177 | } |
178 | |
179 | void rds_tcp_write_space(struct sock *sk) |
180 | { |
181 | void (*write_space)(struct sock *sk); |
182 | struct rds_conn_path *cp; |
183 | struct rds_tcp_connection *tc; |
184 | |
185 | read_lock_bh(&sk->sk_callback_lock); |
186 | cp = sk->sk_user_data; |
187 | if (!cp) { |
188 | write_space = sk->sk_write_space; |
189 | goto out; |
190 | } |
191 | |
192 | tc = cp->cp_transport_data; |
193 | rdsdebug("write_space for tc %p\n" , tc); |
194 | write_space = tc->t_orig_write_space; |
195 | rds_tcp_stats_inc(s_tcp_write_space_calls); |
196 | |
197 | rdsdebug("tcp una %u\n" , rds_tcp_snd_una(tc)); |
198 | tc->t_last_seen_una = rds_tcp_snd_una(tc); |
199 | rds_send_path_drop_acked(cp, ack: rds_tcp_snd_una(tc), is_acked: rds_tcp_is_acked); |
200 | |
201 | rcu_read_lock(); |
202 | if ((refcount_read(r: &sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf && |
203 | !rds_destroy_pending(conn: cp->cp_conn)) |
204 | queue_delayed_work(wq: rds_wq, dwork: &cp->cp_send_w, delay: 0); |
205 | rcu_read_unlock(); |
206 | |
207 | out: |
208 | read_unlock_bh(&sk->sk_callback_lock); |
209 | |
210 | /* |
211 | * write_space is only called when data leaves tcp's send queue if |
212 | * SOCK_NOSPACE is set. We set SOCK_NOSPACE every time we put |
213 | * data in tcp's send queue because we use write_space to parse the |
214 | * sequence numbers and notice that rds messages have been fully |
215 | * received. |
216 | * |
217 | * tcp's write_space clears SOCK_NOSPACE if the send queue has more |
218 | * than a certain amount of space. So we need to set it again *after* |
219 | * we call tcp's write_space or else we might only get called on the |
220 | * first of a series of incoming tcp acks. |
221 | */ |
222 | write_space(sk); |
223 | |
224 | if (sk->sk_socket) |
225 | set_bit(SOCK_NOSPACE, addr: &sk->sk_socket->flags); |
226 | } |
227 | |