1 | /* |
2 | * Copyright (c) 2016 Tom Herbert <tom@herbertland.com> |
3 | * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. |
4 | * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. |
5 | * |
6 | * This software is available to you under a choice of one of two |
7 | * licenses. You may choose to be licensed under the terms of the GNU |
8 | * General Public License (GPL) Version 2, available from the file |
9 | * COPYING in the main directory of this source tree, or the |
10 | * OpenIB.org BSD license below: |
11 | * |
12 | * Redistribution and use in source and binary forms, with or |
13 | * without modification, are permitted provided that the following |
14 | * conditions are met: |
15 | * |
16 | * - Redistributions of source code must retain the above |
17 | * copyright notice, this list of conditions and the following |
18 | * disclaimer. |
19 | * |
20 | * - Redistributions in binary form must reproduce the above |
21 | * copyright notice, this list of conditions and the following |
22 | * disclaimer in the documentation and/or other materials |
23 | * provided with the distribution. |
24 | * |
25 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
26 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
27 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
28 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
29 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
30 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
31 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
32 | * SOFTWARE. |
33 | */ |
34 | |
35 | #ifndef _TLS_INT_H |
36 | #define _TLS_INT_H |
37 | |
38 | #include <asm/byteorder.h> |
39 | #include <linux/types.h> |
40 | #include <linux/skmsg.h> |
41 | #include <net/tls.h> |
42 | #include <net/tls_prot.h> |
43 | |
44 | #define TLS_PAGE_ORDER (min_t(unsigned int, PAGE_ALLOC_COSTLY_ORDER, \ |
45 | TLS_MAX_PAYLOAD_SIZE >> PAGE_SHIFT)) |
46 | |
47 | #define __TLS_INC_STATS(net, field) \ |
48 | __SNMP_INC_STATS((net)->mib.tls_statistics, field) |
49 | #define TLS_INC_STATS(net, field) \ |
50 | SNMP_INC_STATS((net)->mib.tls_statistics, field) |
51 | #define TLS_DEC_STATS(net, field) \ |
52 | SNMP_DEC_STATS((net)->mib.tls_statistics, field) |
53 | |
54 | struct tls_cipher_desc { |
55 | unsigned int nonce; |
56 | unsigned int iv; |
57 | unsigned int key; |
58 | unsigned int salt; |
59 | unsigned int tag; |
60 | unsigned int rec_seq; |
61 | unsigned int iv_offset; |
62 | unsigned int key_offset; |
63 | unsigned int salt_offset; |
64 | unsigned int rec_seq_offset; |
65 | char *cipher_name; |
66 | bool offloadable; |
67 | size_t crypto_info; |
68 | }; |
69 | |
70 | #define TLS_CIPHER_MIN TLS_CIPHER_AES_GCM_128 |
71 | #define TLS_CIPHER_MAX TLS_CIPHER_ARIA_GCM_256 |
72 | extern const struct tls_cipher_desc tls_cipher_desc[TLS_CIPHER_MAX + 1 - TLS_CIPHER_MIN]; |
73 | |
74 | static inline const struct tls_cipher_desc *get_cipher_desc(u16 cipher_type) |
75 | { |
76 | if (cipher_type < TLS_CIPHER_MIN || cipher_type > TLS_CIPHER_MAX) |
77 | return NULL; |
78 | |
79 | return &tls_cipher_desc[cipher_type - TLS_CIPHER_MIN]; |
80 | } |
81 | |
82 | static inline char *crypto_info_iv(struct tls_crypto_info *crypto_info, |
83 | const struct tls_cipher_desc *cipher_desc) |
84 | { |
85 | return (char *)crypto_info + cipher_desc->iv_offset; |
86 | } |
87 | |
88 | static inline char *crypto_info_key(struct tls_crypto_info *crypto_info, |
89 | const struct tls_cipher_desc *cipher_desc) |
90 | { |
91 | return (char *)crypto_info + cipher_desc->key_offset; |
92 | } |
93 | |
94 | static inline char *crypto_info_salt(struct tls_crypto_info *crypto_info, |
95 | const struct tls_cipher_desc *cipher_desc) |
96 | { |
97 | return (char *)crypto_info + cipher_desc->salt_offset; |
98 | } |
99 | |
100 | static inline char *crypto_info_rec_seq(struct tls_crypto_info *crypto_info, |
101 | const struct tls_cipher_desc *cipher_desc) |
102 | { |
103 | return (char *)crypto_info + cipher_desc->rec_seq_offset; |
104 | } |
105 | |
106 | |
107 | /* TLS records are maintained in 'struct tls_rec'. It stores the memory pages |
108 | * allocated or mapped for each TLS record. After encryption, the records are |
109 | * stores in a linked list. |
110 | */ |
111 | struct tls_rec { |
112 | struct list_head list; |
113 | int tx_ready; |
114 | int tx_flags; |
115 | |
116 | struct sk_msg msg_plaintext; |
117 | struct sk_msg msg_encrypted; |
118 | |
119 | /* AAD | msg_plaintext.sg.data | sg_tag */ |
120 | struct scatterlist sg_aead_in[2]; |
121 | /* AAD | msg_encrypted.sg.data (data contains overhead for hdr & iv & tag) */ |
122 | struct scatterlist sg_aead_out[2]; |
123 | |
124 | char content_type; |
125 | struct scatterlist sg_content_type; |
126 | |
127 | struct sock *sk; |
128 | |
129 | char aad_space[TLS_AAD_SPACE_SIZE]; |
130 | u8 iv_data[TLS_MAX_IV_SIZE]; |
131 | struct aead_request aead_req; |
132 | u8 aead_req_ctx[]; |
133 | }; |
134 | |
135 | int __net_init tls_proc_init(struct net *net); |
136 | void __net_exit tls_proc_fini(struct net *net); |
137 | |
138 | struct tls_context *tls_ctx_create(struct sock *sk); |
139 | void tls_ctx_free(struct sock *sk, struct tls_context *ctx); |
140 | void update_sk_prot(struct sock *sk, struct tls_context *ctx); |
141 | |
142 | int wait_on_pending_writer(struct sock *sk, long *timeo); |
143 | void tls_err_abort(struct sock *sk, int err); |
144 | |
145 | int init_prot_info(struct tls_prot_info *prot, |
146 | const struct tls_crypto_info *crypto_info, |
147 | const struct tls_cipher_desc *cipher_desc); |
148 | int tls_set_sw_offload(struct sock *sk, int tx); |
149 | void tls_update_rx_zc_capable(struct tls_context *tls_ctx); |
150 | void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx); |
151 | void tls_sw_strparser_done(struct tls_context *tls_ctx); |
152 | int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); |
153 | void tls_sw_splice_eof(struct socket *sock); |
154 | void tls_sw_cancel_work_tx(struct tls_context *tls_ctx); |
155 | void tls_sw_release_resources_tx(struct sock *sk); |
156 | void tls_sw_free_ctx_tx(struct tls_context *tls_ctx); |
157 | void tls_sw_free_resources_rx(struct sock *sk); |
158 | void tls_sw_release_resources_rx(struct sock *sk); |
159 | void tls_sw_free_ctx_rx(struct tls_context *tls_ctx); |
160 | int tls_sw_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, |
161 | int flags, int *addr_len); |
162 | bool tls_sw_sock_is_readable(struct sock *sk); |
163 | ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, |
164 | struct pipe_inode_info *pipe, |
165 | size_t len, unsigned int flags); |
166 | int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc, |
167 | sk_read_actor_t read_actor); |
168 | |
169 | int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); |
170 | void tls_device_splice_eof(struct socket *sock); |
171 | int tls_tx_records(struct sock *sk, int flags); |
172 | |
173 | void tls_sw_write_space(struct sock *sk, struct tls_context *ctx); |
174 | void tls_device_write_space(struct sock *sk, struct tls_context *ctx); |
175 | |
176 | int tls_process_cmsg(struct sock *sk, struct msghdr *msg, |
177 | unsigned char *record_type); |
178 | int decrypt_skb(struct sock *sk, struct scatterlist *sgout); |
179 | |
180 | int tls_sw_fallback_init(struct sock *sk, |
181 | struct tls_offload_context_tx *offload_ctx, |
182 | struct tls_crypto_info *crypto_info); |
183 | |
184 | int tls_strp_dev_init(void); |
185 | void tls_strp_dev_exit(void); |
186 | |
187 | void tls_strp_done(struct tls_strparser *strp); |
188 | void tls_strp_stop(struct tls_strparser *strp); |
189 | int tls_strp_init(struct tls_strparser *strp, struct sock *sk); |
190 | void tls_strp_data_ready(struct tls_strparser *strp); |
191 | |
192 | void tls_strp_check_rcv(struct tls_strparser *strp); |
193 | void tls_strp_msg_done(struct tls_strparser *strp); |
194 | |
195 | int tls_rx_msg_size(struct tls_strparser *strp, struct sk_buff *skb); |
196 | void tls_rx_msg_ready(struct tls_strparser *strp); |
197 | |
198 | void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh); |
199 | int tls_strp_msg_cow(struct tls_sw_context_rx *ctx); |
200 | struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx); |
201 | int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst); |
202 | |
203 | static inline struct tls_msg *tls_msg(struct sk_buff *skb) |
204 | { |
205 | struct sk_skb_cb *scb = (struct sk_skb_cb *)skb->cb; |
206 | |
207 | return &scb->tls; |
208 | } |
209 | |
210 | static inline struct sk_buff *tls_strp_msg(struct tls_sw_context_rx *ctx) |
211 | { |
212 | DEBUG_NET_WARN_ON_ONCE(!ctx->strp.msg_ready || !ctx->strp.anchor->len); |
213 | return ctx->strp.anchor; |
214 | } |
215 | |
216 | static inline bool tls_strp_msg_ready(struct tls_sw_context_rx *ctx) |
217 | { |
218 | return ctx->strp.msg_ready; |
219 | } |
220 | |
221 | static inline bool tls_strp_msg_mixed_decrypted(struct tls_sw_context_rx *ctx) |
222 | { |
223 | return ctx->strp.mixed_decrypted; |
224 | } |
225 | |
226 | #ifdef CONFIG_TLS_DEVICE |
227 | int tls_device_init(void); |
228 | void tls_device_cleanup(void); |
229 | int tls_set_device_offload(struct sock *sk); |
230 | void tls_device_free_resources_tx(struct sock *sk); |
231 | int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx); |
232 | void tls_device_offload_cleanup_rx(struct sock *sk); |
233 | void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq); |
234 | int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx); |
235 | #else |
236 | static inline int tls_device_init(void) { return 0; } |
237 | static inline void tls_device_cleanup(void) {} |
238 | |
239 | static inline int |
240 | tls_set_device_offload(struct sock *sk) |
241 | { |
242 | return -EOPNOTSUPP; |
243 | } |
244 | |
245 | static inline void tls_device_free_resources_tx(struct sock *sk) {} |
246 | |
247 | static inline int |
248 | tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx) |
249 | { |
250 | return -EOPNOTSUPP; |
251 | } |
252 | |
253 | static inline void tls_device_offload_cleanup_rx(struct sock *sk) {} |
254 | static inline void |
255 | tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) {} |
256 | |
257 | static inline int |
258 | tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx) |
259 | { |
260 | return 0; |
261 | } |
262 | #endif |
263 | |
264 | int tls_push_sg(struct sock *sk, struct tls_context *ctx, |
265 | struct scatterlist *sg, u16 first_offset, |
266 | int flags); |
267 | int tls_push_partial_record(struct sock *sk, struct tls_context *ctx, |
268 | int flags); |
269 | void tls_free_partial_record(struct sock *sk, struct tls_context *ctx); |
270 | |
271 | static inline bool tls_is_partially_sent_record(struct tls_context *ctx) |
272 | { |
273 | return !!ctx->partially_sent_record; |
274 | } |
275 | |
276 | static inline bool tls_is_pending_open_record(struct tls_context *tls_ctx) |
277 | { |
278 | return tls_ctx->pending_open_record_frags; |
279 | } |
280 | |
281 | static inline bool tls_bigint_increment(unsigned char *seq, int len) |
282 | { |
283 | int i; |
284 | |
285 | for (i = len - 1; i >= 0; i--) { |
286 | ++seq[i]; |
287 | if (seq[i] != 0) |
288 | break; |
289 | } |
290 | |
291 | return (i == -1); |
292 | } |
293 | |
294 | static inline void tls_bigint_subtract(unsigned char *seq, int n) |
295 | { |
296 | u64 rcd_sn; |
297 | __be64 *p; |
298 | |
299 | BUILD_BUG_ON(TLS_MAX_REC_SEQ_SIZE != 8); |
300 | |
301 | p = (__be64 *)seq; |
302 | rcd_sn = be64_to_cpu(*p); |
303 | *p = cpu_to_be64(rcd_sn - n); |
304 | } |
305 | |
306 | static inline void |
307 | tls_advance_record_sn(struct sock *sk, struct tls_prot_info *prot, |
308 | struct cipher_context *ctx) |
309 | { |
310 | if (tls_bigint_increment(seq: ctx->rec_seq, len: prot->rec_seq_size)) |
311 | tls_err_abort(sk, err: -EBADMSG); |
312 | |
313 | if (prot->version != TLS_1_3_VERSION && |
314 | prot->cipher_type != TLS_CIPHER_CHACHA20_POLY1305) |
315 | tls_bigint_increment(seq: ctx->iv + prot->salt_size, |
316 | len: prot->iv_size); |
317 | } |
318 | |
319 | static inline void |
320 | tls_xor_iv_with_seq(struct tls_prot_info *prot, char *iv, char *seq) |
321 | { |
322 | int i; |
323 | |
324 | if (prot->version == TLS_1_3_VERSION || |
325 | prot->cipher_type == TLS_CIPHER_CHACHA20_POLY1305) { |
326 | for (i = 0; i < 8; i++) |
327 | iv[i + 4] ^= seq[i]; |
328 | } |
329 | } |
330 | |
331 | static inline void |
332 | tls_fill_prepend(struct tls_context *ctx, char *buf, size_t plaintext_len, |
333 | unsigned char record_type) |
334 | { |
335 | struct tls_prot_info *prot = &ctx->prot_info; |
336 | size_t pkt_len, iv_size = prot->iv_size; |
337 | |
338 | pkt_len = plaintext_len + prot->tag_size; |
339 | if (prot->version != TLS_1_3_VERSION && |
340 | prot->cipher_type != TLS_CIPHER_CHACHA20_POLY1305) { |
341 | pkt_len += iv_size; |
342 | |
343 | memcpy(buf + TLS_NONCE_OFFSET, |
344 | ctx->tx.iv + prot->salt_size, iv_size); |
345 | } |
346 | |
347 | /* we cover nonce explicit here as well, so buf should be of |
348 | * size KTLS_DTLS_HEADER_SIZE + KTLS_DTLS_NONCE_EXPLICIT_SIZE |
349 | */ |
350 | buf[0] = prot->version == TLS_1_3_VERSION ? |
351 | TLS_RECORD_TYPE_DATA : record_type; |
352 | /* Note that VERSION must be TLS_1_2 for both TLS1.2 and TLS1.3 */ |
353 | buf[1] = TLS_1_2_VERSION_MINOR; |
354 | buf[2] = TLS_1_2_VERSION_MAJOR; |
355 | /* we can use IV for nonce explicit according to spec */ |
356 | buf[3] = pkt_len >> 8; |
357 | buf[4] = pkt_len & 0xFF; |
358 | } |
359 | |
360 | static inline |
361 | void tls_make_aad(char *buf, size_t size, char *record_sequence, |
362 | unsigned char record_type, struct tls_prot_info *prot) |
363 | { |
364 | if (prot->version != TLS_1_3_VERSION) { |
365 | memcpy(buf, record_sequence, prot->rec_seq_size); |
366 | buf += 8; |
367 | } else { |
368 | size += prot->tag_size; |
369 | } |
370 | |
371 | buf[0] = prot->version == TLS_1_3_VERSION ? |
372 | TLS_RECORD_TYPE_DATA : record_type; |
373 | buf[1] = TLS_1_2_VERSION_MAJOR; |
374 | buf[2] = TLS_1_2_VERSION_MINOR; |
375 | buf[3] = size >> 8; |
376 | buf[4] = size & 0xFF; |
377 | } |
378 | |
379 | #endif |
380 | |