1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /******************************************************************************* |
3 | * This file contains the iSCSI Target DataIN value generation functions. |
4 | * |
5 | * (c) Copyright 2007-2013 Datera, Inc. |
6 | * |
7 | * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> |
8 | * |
9 | ******************************************************************************/ |
10 | |
11 | #include <linux/slab.h> |
12 | #include <scsi/iscsi_proto.h> |
13 | #include <target/iscsi/iscsi_target_core.h> |
14 | #include "iscsi_target_seq_pdu_list.h" |
15 | #include "iscsi_target_erl1.h" |
16 | #include "iscsi_target_util.h" |
17 | #include "iscsi_target.h" |
18 | #include "iscsi_target_datain_values.h" |
19 | |
20 | struct iscsi_datain_req *iscsit_allocate_datain_req(void) |
21 | { |
22 | struct iscsi_datain_req *dr; |
23 | |
24 | dr = kmem_cache_zalloc(k: lio_dr_cache, GFP_ATOMIC); |
25 | if (!dr) { |
26 | pr_err("Unable to allocate memory for" |
27 | " struct iscsi_datain_req\n" ); |
28 | return NULL; |
29 | } |
30 | INIT_LIST_HEAD(list: &dr->cmd_datain_node); |
31 | |
32 | return dr; |
33 | } |
34 | |
35 | void iscsit_attach_datain_req(struct iscsit_cmd *cmd, struct iscsi_datain_req *dr) |
36 | { |
37 | spin_lock(lock: &cmd->datain_lock); |
38 | list_add_tail(new: &dr->cmd_datain_node, head: &cmd->datain_list); |
39 | spin_unlock(lock: &cmd->datain_lock); |
40 | } |
41 | |
42 | void iscsit_free_datain_req(struct iscsit_cmd *cmd, struct iscsi_datain_req *dr) |
43 | { |
44 | spin_lock(lock: &cmd->datain_lock); |
45 | list_del(entry: &dr->cmd_datain_node); |
46 | spin_unlock(lock: &cmd->datain_lock); |
47 | |
48 | kmem_cache_free(s: lio_dr_cache, objp: dr); |
49 | } |
50 | |
51 | void iscsit_free_all_datain_reqs(struct iscsit_cmd *cmd) |
52 | { |
53 | struct iscsi_datain_req *dr, *dr_tmp; |
54 | |
55 | spin_lock(lock: &cmd->datain_lock); |
56 | list_for_each_entry_safe(dr, dr_tmp, &cmd->datain_list, cmd_datain_node) { |
57 | list_del(entry: &dr->cmd_datain_node); |
58 | kmem_cache_free(s: lio_dr_cache, objp: dr); |
59 | } |
60 | spin_unlock(lock: &cmd->datain_lock); |
61 | } |
62 | |
63 | struct iscsi_datain_req *iscsit_get_datain_req(struct iscsit_cmd *cmd) |
64 | { |
65 | if (list_empty(head: &cmd->datain_list)) { |
66 | pr_err("cmd->datain_list is empty for ITT:" |
67 | " 0x%08x\n" , cmd->init_task_tag); |
68 | return NULL; |
69 | } |
70 | |
71 | return list_first_entry(&cmd->datain_list, struct iscsi_datain_req, |
72 | cmd_datain_node); |
73 | } |
74 | |
75 | /* |
76 | * For Normal and Recovery DataSequenceInOrder=Yes and DataPDUInOrder=Yes. |
77 | */ |
78 | static struct iscsi_datain_req *iscsit_set_datain_values_yes_and_yes( |
79 | struct iscsit_cmd *cmd, |
80 | struct iscsi_datain *datain) |
81 | { |
82 | u32 next_burst_len, read_data_done, read_data_left; |
83 | struct iscsit_conn *conn = cmd->conn; |
84 | struct iscsi_datain_req *dr; |
85 | |
86 | dr = iscsit_get_datain_req(cmd); |
87 | if (!dr) |
88 | return NULL; |
89 | |
90 | if (dr->recovery && dr->generate_recovery_values) { |
91 | if (iscsit_create_recovery_datain_values_datasequenceinorder_yes( |
92 | cmd, dr) < 0) |
93 | return NULL; |
94 | |
95 | dr->generate_recovery_values = 0; |
96 | } |
97 | |
98 | next_burst_len = (!dr->recovery) ? |
99 | cmd->next_burst_len : dr->next_burst_len; |
100 | read_data_done = (!dr->recovery) ? |
101 | cmd->read_data_done : dr->read_data_done; |
102 | |
103 | read_data_left = (cmd->se_cmd.data_length - read_data_done); |
104 | if (!read_data_left) { |
105 | pr_err("ITT: 0x%08x read_data_left is zero!\n" , |
106 | cmd->init_task_tag); |
107 | return NULL; |
108 | } |
109 | |
110 | if ((read_data_left <= conn->conn_ops->MaxRecvDataSegmentLength) && |
111 | (read_data_left <= (conn->sess->sess_ops->MaxBurstLength - |
112 | next_burst_len))) { |
113 | datain->length = read_data_left; |
114 | |
115 | datain->flags |= (ISCSI_FLAG_CMD_FINAL | ISCSI_FLAG_DATA_STATUS); |
116 | if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) |
117 | datain->flags |= ISCSI_FLAG_DATA_ACK; |
118 | } else { |
119 | if ((next_burst_len + |
120 | conn->conn_ops->MaxRecvDataSegmentLength) < |
121 | conn->sess->sess_ops->MaxBurstLength) { |
122 | datain->length = |
123 | conn->conn_ops->MaxRecvDataSegmentLength; |
124 | next_burst_len += datain->length; |
125 | } else { |
126 | datain->length = (conn->sess->sess_ops->MaxBurstLength - |
127 | next_burst_len); |
128 | next_burst_len = 0; |
129 | |
130 | datain->flags |= ISCSI_FLAG_CMD_FINAL; |
131 | if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) |
132 | datain->flags |= ISCSI_FLAG_DATA_ACK; |
133 | } |
134 | } |
135 | |
136 | datain->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++; |
137 | datain->offset = read_data_done; |
138 | |
139 | if (!dr->recovery) { |
140 | cmd->next_burst_len = next_burst_len; |
141 | cmd->read_data_done += datain->length; |
142 | } else { |
143 | dr->next_burst_len = next_burst_len; |
144 | dr->read_data_done += datain->length; |
145 | } |
146 | |
147 | if (!dr->recovery) { |
148 | if (datain->flags & ISCSI_FLAG_DATA_STATUS) |
149 | dr->dr_complete = DATAIN_COMPLETE_NORMAL; |
150 | |
151 | return dr; |
152 | } |
153 | |
154 | if (!dr->runlength) { |
155 | if (datain->flags & ISCSI_FLAG_DATA_STATUS) { |
156 | dr->dr_complete = |
157 | (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? |
158 | DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : |
159 | DATAIN_COMPLETE_CONNECTION_RECOVERY; |
160 | } |
161 | } else { |
162 | if ((dr->begrun + dr->runlength) == dr->data_sn) { |
163 | dr->dr_complete = |
164 | (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? |
165 | DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : |
166 | DATAIN_COMPLETE_CONNECTION_RECOVERY; |
167 | } |
168 | } |
169 | |
170 | return dr; |
171 | } |
172 | |
173 | /* |
174 | * For Normal and Recovery DataSequenceInOrder=No and DataPDUInOrder=Yes. |
175 | */ |
176 | static struct iscsi_datain_req *iscsit_set_datain_values_no_and_yes( |
177 | struct iscsit_cmd *cmd, |
178 | struct iscsi_datain *datain) |
179 | { |
180 | u32 offset, read_data_done, read_data_left, seq_send_order; |
181 | struct iscsit_conn *conn = cmd->conn; |
182 | struct iscsi_datain_req *dr; |
183 | struct iscsi_seq *seq; |
184 | |
185 | dr = iscsit_get_datain_req(cmd); |
186 | if (!dr) |
187 | return NULL; |
188 | |
189 | if (dr->recovery && dr->generate_recovery_values) { |
190 | if (iscsit_create_recovery_datain_values_datasequenceinorder_no( |
191 | cmd, dr) < 0) |
192 | return NULL; |
193 | |
194 | dr->generate_recovery_values = 0; |
195 | } |
196 | |
197 | read_data_done = (!dr->recovery) ? |
198 | cmd->read_data_done : dr->read_data_done; |
199 | seq_send_order = (!dr->recovery) ? |
200 | cmd->seq_send_order : dr->seq_send_order; |
201 | |
202 | read_data_left = (cmd->se_cmd.data_length - read_data_done); |
203 | if (!read_data_left) { |
204 | pr_err("ITT: 0x%08x read_data_left is zero!\n" , |
205 | cmd->init_task_tag); |
206 | return NULL; |
207 | } |
208 | |
209 | seq = iscsit_get_seq_holder_for_datain(cmd, seq_send_order); |
210 | if (!seq) |
211 | return NULL; |
212 | |
213 | seq->sent = 1; |
214 | |
215 | if (!dr->recovery && !seq->next_burst_len) |
216 | seq->first_datasn = cmd->data_sn; |
217 | |
218 | offset = (seq->offset + seq->next_burst_len); |
219 | |
220 | if ((offset + conn->conn_ops->MaxRecvDataSegmentLength) >= |
221 | cmd->se_cmd.data_length) { |
222 | datain->length = (cmd->se_cmd.data_length - offset); |
223 | datain->offset = offset; |
224 | |
225 | datain->flags |= ISCSI_FLAG_CMD_FINAL; |
226 | if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) |
227 | datain->flags |= ISCSI_FLAG_DATA_ACK; |
228 | |
229 | seq->next_burst_len = 0; |
230 | seq_send_order++; |
231 | } else { |
232 | if ((seq->next_burst_len + |
233 | conn->conn_ops->MaxRecvDataSegmentLength) < |
234 | conn->sess->sess_ops->MaxBurstLength) { |
235 | datain->length = |
236 | conn->conn_ops->MaxRecvDataSegmentLength; |
237 | datain->offset = (seq->offset + seq->next_burst_len); |
238 | |
239 | seq->next_burst_len += datain->length; |
240 | } else { |
241 | datain->length = (conn->sess->sess_ops->MaxBurstLength - |
242 | seq->next_burst_len); |
243 | datain->offset = (seq->offset + seq->next_burst_len); |
244 | |
245 | datain->flags |= ISCSI_FLAG_CMD_FINAL; |
246 | if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) |
247 | datain->flags |= ISCSI_FLAG_DATA_ACK; |
248 | |
249 | seq->next_burst_len = 0; |
250 | seq_send_order++; |
251 | } |
252 | } |
253 | |
254 | if ((read_data_done + datain->length) == cmd->se_cmd.data_length) |
255 | datain->flags |= ISCSI_FLAG_DATA_STATUS; |
256 | |
257 | datain->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++; |
258 | if (!dr->recovery) { |
259 | cmd->seq_send_order = seq_send_order; |
260 | cmd->read_data_done += datain->length; |
261 | } else { |
262 | dr->seq_send_order = seq_send_order; |
263 | dr->read_data_done += datain->length; |
264 | } |
265 | |
266 | if (!dr->recovery) { |
267 | if (datain->flags & ISCSI_FLAG_CMD_FINAL) |
268 | seq->last_datasn = datain->data_sn; |
269 | if (datain->flags & ISCSI_FLAG_DATA_STATUS) |
270 | dr->dr_complete = DATAIN_COMPLETE_NORMAL; |
271 | |
272 | return dr; |
273 | } |
274 | |
275 | if (!dr->runlength) { |
276 | if (datain->flags & ISCSI_FLAG_DATA_STATUS) { |
277 | dr->dr_complete = |
278 | (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? |
279 | DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : |
280 | DATAIN_COMPLETE_CONNECTION_RECOVERY; |
281 | } |
282 | } else { |
283 | if ((dr->begrun + dr->runlength) == dr->data_sn) { |
284 | dr->dr_complete = |
285 | (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? |
286 | DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : |
287 | DATAIN_COMPLETE_CONNECTION_RECOVERY; |
288 | } |
289 | } |
290 | |
291 | return dr; |
292 | } |
293 | |
294 | /* |
295 | * For Normal and Recovery DataSequenceInOrder=Yes and DataPDUInOrder=No. |
296 | */ |
297 | static struct iscsi_datain_req *iscsit_set_datain_values_yes_and_no( |
298 | struct iscsit_cmd *cmd, |
299 | struct iscsi_datain *datain) |
300 | { |
301 | u32 next_burst_len, read_data_done, read_data_left; |
302 | struct iscsit_conn *conn = cmd->conn; |
303 | struct iscsi_datain_req *dr; |
304 | struct iscsi_pdu *pdu; |
305 | |
306 | dr = iscsit_get_datain_req(cmd); |
307 | if (!dr) |
308 | return NULL; |
309 | |
310 | if (dr->recovery && dr->generate_recovery_values) { |
311 | if (iscsit_create_recovery_datain_values_datasequenceinorder_yes( |
312 | cmd, dr) < 0) |
313 | return NULL; |
314 | |
315 | dr->generate_recovery_values = 0; |
316 | } |
317 | |
318 | next_burst_len = (!dr->recovery) ? |
319 | cmd->next_burst_len : dr->next_burst_len; |
320 | read_data_done = (!dr->recovery) ? |
321 | cmd->read_data_done : dr->read_data_done; |
322 | |
323 | read_data_left = (cmd->se_cmd.data_length - read_data_done); |
324 | if (!read_data_left) { |
325 | pr_err("ITT: 0x%08x read_data_left is zero!\n" , |
326 | cmd->init_task_tag); |
327 | return dr; |
328 | } |
329 | |
330 | pdu = iscsit_get_pdu_holder_for_seq(cmd, NULL); |
331 | if (!pdu) |
332 | return dr; |
333 | |
334 | if ((read_data_done + pdu->length) == cmd->se_cmd.data_length) { |
335 | pdu->flags |= (ISCSI_FLAG_CMD_FINAL | ISCSI_FLAG_DATA_STATUS); |
336 | if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) |
337 | pdu->flags |= ISCSI_FLAG_DATA_ACK; |
338 | |
339 | next_burst_len = 0; |
340 | } else { |
341 | if ((next_burst_len + conn->conn_ops->MaxRecvDataSegmentLength) < |
342 | conn->sess->sess_ops->MaxBurstLength) |
343 | next_burst_len += pdu->length; |
344 | else { |
345 | pdu->flags |= ISCSI_FLAG_CMD_FINAL; |
346 | if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) |
347 | pdu->flags |= ISCSI_FLAG_DATA_ACK; |
348 | |
349 | next_burst_len = 0; |
350 | } |
351 | } |
352 | |
353 | pdu->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++; |
354 | if (!dr->recovery) { |
355 | cmd->next_burst_len = next_burst_len; |
356 | cmd->read_data_done += pdu->length; |
357 | } else { |
358 | dr->next_burst_len = next_burst_len; |
359 | dr->read_data_done += pdu->length; |
360 | } |
361 | |
362 | datain->flags = pdu->flags; |
363 | datain->length = pdu->length; |
364 | datain->offset = pdu->offset; |
365 | datain->data_sn = pdu->data_sn; |
366 | |
367 | if (!dr->recovery) { |
368 | if (datain->flags & ISCSI_FLAG_DATA_STATUS) |
369 | dr->dr_complete = DATAIN_COMPLETE_NORMAL; |
370 | |
371 | return dr; |
372 | } |
373 | |
374 | if (!dr->runlength) { |
375 | if (datain->flags & ISCSI_FLAG_DATA_STATUS) { |
376 | dr->dr_complete = |
377 | (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? |
378 | DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : |
379 | DATAIN_COMPLETE_CONNECTION_RECOVERY; |
380 | } |
381 | } else { |
382 | if ((dr->begrun + dr->runlength) == dr->data_sn) { |
383 | dr->dr_complete = |
384 | (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? |
385 | DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : |
386 | DATAIN_COMPLETE_CONNECTION_RECOVERY; |
387 | } |
388 | } |
389 | |
390 | return dr; |
391 | } |
392 | |
393 | /* |
394 | * For Normal and Recovery DataSequenceInOrder=No and DataPDUInOrder=No. |
395 | */ |
396 | static struct iscsi_datain_req *iscsit_set_datain_values_no_and_no( |
397 | struct iscsit_cmd *cmd, |
398 | struct iscsi_datain *datain) |
399 | { |
400 | u32 read_data_done, read_data_left, seq_send_order; |
401 | struct iscsit_conn *conn = cmd->conn; |
402 | struct iscsi_datain_req *dr; |
403 | struct iscsi_pdu *pdu; |
404 | struct iscsi_seq *seq = NULL; |
405 | |
406 | dr = iscsit_get_datain_req(cmd); |
407 | if (!dr) |
408 | return NULL; |
409 | |
410 | if (dr->recovery && dr->generate_recovery_values) { |
411 | if (iscsit_create_recovery_datain_values_datasequenceinorder_no( |
412 | cmd, dr) < 0) |
413 | return NULL; |
414 | |
415 | dr->generate_recovery_values = 0; |
416 | } |
417 | |
418 | read_data_done = (!dr->recovery) ? |
419 | cmd->read_data_done : dr->read_data_done; |
420 | seq_send_order = (!dr->recovery) ? |
421 | cmd->seq_send_order : dr->seq_send_order; |
422 | |
423 | read_data_left = (cmd->se_cmd.data_length - read_data_done); |
424 | if (!read_data_left) { |
425 | pr_err("ITT: 0x%08x read_data_left is zero!\n" , |
426 | cmd->init_task_tag); |
427 | return NULL; |
428 | } |
429 | |
430 | seq = iscsit_get_seq_holder_for_datain(cmd, seq_send_order); |
431 | if (!seq) |
432 | return NULL; |
433 | |
434 | seq->sent = 1; |
435 | |
436 | if (!dr->recovery && !seq->next_burst_len) |
437 | seq->first_datasn = cmd->data_sn; |
438 | |
439 | pdu = iscsit_get_pdu_holder_for_seq(cmd, seq); |
440 | if (!pdu) |
441 | return NULL; |
442 | |
443 | if (seq->pdu_send_order == seq->pdu_count) { |
444 | pdu->flags |= ISCSI_FLAG_CMD_FINAL; |
445 | if (conn->sess->sess_ops->ErrorRecoveryLevel > 0) |
446 | pdu->flags |= ISCSI_FLAG_DATA_ACK; |
447 | |
448 | seq->next_burst_len = 0; |
449 | seq_send_order++; |
450 | } else |
451 | seq->next_burst_len += pdu->length; |
452 | |
453 | if ((read_data_done + pdu->length) == cmd->se_cmd.data_length) |
454 | pdu->flags |= ISCSI_FLAG_DATA_STATUS; |
455 | |
456 | pdu->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++; |
457 | if (!dr->recovery) { |
458 | cmd->seq_send_order = seq_send_order; |
459 | cmd->read_data_done += pdu->length; |
460 | } else { |
461 | dr->seq_send_order = seq_send_order; |
462 | dr->read_data_done += pdu->length; |
463 | } |
464 | |
465 | datain->flags = pdu->flags; |
466 | datain->length = pdu->length; |
467 | datain->offset = pdu->offset; |
468 | datain->data_sn = pdu->data_sn; |
469 | |
470 | if (!dr->recovery) { |
471 | if (datain->flags & ISCSI_FLAG_CMD_FINAL) |
472 | seq->last_datasn = datain->data_sn; |
473 | if (datain->flags & ISCSI_FLAG_DATA_STATUS) |
474 | dr->dr_complete = DATAIN_COMPLETE_NORMAL; |
475 | |
476 | return dr; |
477 | } |
478 | |
479 | if (!dr->runlength) { |
480 | if (datain->flags & ISCSI_FLAG_DATA_STATUS) { |
481 | dr->dr_complete = |
482 | (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? |
483 | DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : |
484 | DATAIN_COMPLETE_CONNECTION_RECOVERY; |
485 | } |
486 | } else { |
487 | if ((dr->begrun + dr->runlength) == dr->data_sn) { |
488 | dr->dr_complete = |
489 | (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ? |
490 | DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY : |
491 | DATAIN_COMPLETE_CONNECTION_RECOVERY; |
492 | } |
493 | } |
494 | |
495 | return dr; |
496 | } |
497 | |
498 | struct iscsi_datain_req *iscsit_get_datain_values( |
499 | struct iscsit_cmd *cmd, |
500 | struct iscsi_datain *datain) |
501 | { |
502 | struct iscsit_conn *conn = cmd->conn; |
503 | |
504 | if (conn->sess->sess_ops->DataSequenceInOrder && |
505 | conn->sess->sess_ops->DataPDUInOrder) |
506 | return iscsit_set_datain_values_yes_and_yes(cmd, datain); |
507 | else if (!conn->sess->sess_ops->DataSequenceInOrder && |
508 | conn->sess->sess_ops->DataPDUInOrder) |
509 | return iscsit_set_datain_values_no_and_yes(cmd, datain); |
510 | else if (conn->sess->sess_ops->DataSequenceInOrder && |
511 | !conn->sess->sess_ops->DataPDUInOrder) |
512 | return iscsit_set_datain_values_yes_and_no(cmd, datain); |
513 | else if (!conn->sess->sess_ops->DataSequenceInOrder && |
514 | !conn->sess->sess_ops->DataPDUInOrder) |
515 | return iscsit_set_datain_values_no_and_no(cmd, datain); |
516 | |
517 | return NULL; |
518 | } |
519 | EXPORT_SYMBOL(iscsit_get_datain_values); |
520 | |