1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /******************************************************************************* |
3 | * This file contains error recovery level one used by the iSCSI Target driver. |
4 | * |
5 | * (c) Copyright 2007-2013 Datera, Inc. |
6 | * |
7 | * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> |
8 | * |
9 | ******************************************************************************/ |
10 | |
11 | #include <linux/list.h> |
12 | #include <linux/slab.h> |
13 | #include <scsi/iscsi_proto.h> |
14 | #include <target/target_core_base.h> |
15 | #include <target/target_core_fabric.h> |
16 | #include <target/iscsi/iscsi_transport.h> |
17 | |
18 | #include <target/iscsi/iscsi_target_core.h> |
19 | #include "iscsi_target_seq_pdu_list.h" |
20 | #include "iscsi_target_datain_values.h" |
21 | #include "iscsi_target_device.h" |
22 | #include "iscsi_target_tpg.h" |
23 | #include "iscsi_target_util.h" |
24 | #include "iscsi_target_erl0.h" |
25 | #include "iscsi_target_erl1.h" |
26 | #include "iscsi_target_erl2.h" |
27 | #include "iscsi_target.h" |
28 | |
29 | #define OFFLOAD_BUF_SIZE 32768U |
30 | |
31 | /* |
32 | * Used to dump excess datain payload for certain error recovery |
33 | * situations. Receive in OFFLOAD_BUF_SIZE max of datain per rx_data(). |
34 | * |
35 | * dump_padding_digest denotes if padding and data digests need |
36 | * to be dumped. |
37 | */ |
38 | int iscsit_dump_data_payload( |
39 | struct iscsit_conn *conn, |
40 | u32 buf_len, |
41 | int dump_padding_digest) |
42 | { |
43 | char *buf; |
44 | int ret = DATAOUT_WITHIN_COMMAND_RECOVERY, rx_got; |
45 | u32 length, offset = 0, size; |
46 | struct kvec iov; |
47 | |
48 | if (conn->sess->sess_ops->RDMAExtensions) |
49 | return 0; |
50 | |
51 | if (dump_padding_digest) { |
52 | buf_len = ALIGN(buf_len, 4); |
53 | if (conn->conn_ops->DataDigest) |
54 | buf_len += ISCSI_CRC_LEN; |
55 | } |
56 | |
57 | length = min(buf_len, OFFLOAD_BUF_SIZE); |
58 | |
59 | buf = kzalloc(size: length, GFP_ATOMIC); |
60 | if (!buf) { |
61 | pr_err("Unable to allocate %u bytes for offload" |
62 | " buffer.\n" , length); |
63 | return -1; |
64 | } |
65 | memset(&iov, 0, sizeof(struct kvec)); |
66 | |
67 | while (offset < buf_len) { |
68 | size = min(buf_len - offset, length); |
69 | |
70 | iov.iov_len = size; |
71 | iov.iov_base = buf; |
72 | |
73 | rx_got = rx_data(conn, &iov, 1, size); |
74 | if (rx_got != size) { |
75 | ret = DATAOUT_CANNOT_RECOVER; |
76 | break; |
77 | } |
78 | |
79 | offset += size; |
80 | } |
81 | |
82 | kfree(objp: buf); |
83 | return ret; |
84 | } |
85 | |
86 | /* |
87 | * Used for retransmitting R2Ts from a R2T SNACK request. |
88 | */ |
89 | static int iscsit_send_recovery_r2t_for_snack( |
90 | struct iscsit_cmd *cmd, |
91 | struct iscsi_r2t *r2t) |
92 | { |
93 | /* |
94 | * If the struct iscsi_r2t has not been sent yet, we can safely |
95 | * ignore retransmission |
96 | * of the R2TSN in question. |
97 | */ |
98 | spin_lock_bh(lock: &cmd->r2t_lock); |
99 | if (!r2t->sent_r2t) { |
100 | spin_unlock_bh(lock: &cmd->r2t_lock); |
101 | return 0; |
102 | } |
103 | r2t->sent_r2t = 0; |
104 | spin_unlock_bh(lock: &cmd->r2t_lock); |
105 | |
106 | iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T); |
107 | |
108 | return 0; |
109 | } |
110 | |
111 | static int iscsit_handle_r2t_snack( |
112 | struct iscsit_cmd *cmd, |
113 | unsigned char *buf, |
114 | u32 begrun, |
115 | u32 runlength) |
116 | { |
117 | u32 last_r2tsn; |
118 | struct iscsi_r2t *r2t; |
119 | |
120 | /* |
121 | * Make sure the initiator is not requesting retransmission |
122 | * of R2TSNs already acknowledged by a TMR TASK_REASSIGN. |
123 | */ |
124 | if ((cmd->cmd_flags & ICF_GOT_DATACK_SNACK) && |
125 | (begrun <= cmd->acked_data_sn)) { |
126 | pr_err("ITT: 0x%08x, R2T SNACK requesting" |
127 | " retransmission of R2TSN: 0x%08x to 0x%08x but already" |
128 | " acked to R2TSN: 0x%08x by TMR TASK_REASSIGN," |
129 | " protocol error.\n" , cmd->init_task_tag, begrun, |
130 | (begrun + runlength), cmd->acked_data_sn); |
131 | |
132 | return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf); |
133 | } |
134 | |
135 | if (runlength) { |
136 | if ((begrun + runlength) > cmd->r2t_sn) { |
137 | pr_err("Command ITT: 0x%08x received R2T SNACK" |
138 | " with BegRun: 0x%08x, RunLength: 0x%08x, exceeds" |
139 | " current R2TSN: 0x%08x, protocol error.\n" , |
140 | cmd->init_task_tag, begrun, runlength, cmd->r2t_sn); |
141 | return iscsit_reject_cmd(cmd, |
142 | ISCSI_REASON_BOOKMARK_INVALID, buf); |
143 | } |
144 | last_r2tsn = (begrun + runlength); |
145 | } else |
146 | last_r2tsn = cmd->r2t_sn; |
147 | |
148 | while (begrun < last_r2tsn) { |
149 | r2t = iscsit_get_holder_for_r2tsn(cmd, begrun); |
150 | if (!r2t) |
151 | return -1; |
152 | if (iscsit_send_recovery_r2t_for_snack(cmd, r2t) < 0) |
153 | return -1; |
154 | |
155 | begrun++; |
156 | } |
157 | |
158 | return 0; |
159 | } |
160 | |
161 | /* |
162 | * Generates Offsets and NextBurstLength based on Begrun and Runlength |
163 | * carried in a Data SNACK or ExpDataSN in TMR TASK_REASSIGN. |
164 | * |
165 | * For DataSequenceInOrder=Yes and DataPDUInOrder=[Yes,No] only. |
166 | * |
167 | * FIXME: How is this handled for a RData SNACK? |
168 | */ |
169 | int iscsit_create_recovery_datain_values_datasequenceinorder_yes( |
170 | struct iscsit_cmd *cmd, |
171 | struct iscsi_datain_req *dr) |
172 | { |
173 | u32 data_sn = 0, data_sn_count = 0; |
174 | u32 pdu_start = 0, seq_no = 0; |
175 | u32 begrun = dr->begrun; |
176 | struct iscsit_conn *conn = cmd->conn; |
177 | |
178 | while (begrun > data_sn++) { |
179 | data_sn_count++; |
180 | if ((dr->next_burst_len + |
181 | conn->conn_ops->MaxRecvDataSegmentLength) < |
182 | conn->sess->sess_ops->MaxBurstLength) { |
183 | dr->read_data_done += |
184 | conn->conn_ops->MaxRecvDataSegmentLength; |
185 | dr->next_burst_len += |
186 | conn->conn_ops->MaxRecvDataSegmentLength; |
187 | } else { |
188 | dr->read_data_done += |
189 | (conn->sess->sess_ops->MaxBurstLength - |
190 | dr->next_burst_len); |
191 | dr->next_burst_len = 0; |
192 | pdu_start += data_sn_count; |
193 | data_sn_count = 0; |
194 | seq_no++; |
195 | } |
196 | } |
197 | |
198 | if (!conn->sess->sess_ops->DataPDUInOrder) { |
199 | cmd->seq_no = seq_no; |
200 | cmd->pdu_start = pdu_start; |
201 | cmd->pdu_send_order = data_sn_count; |
202 | } |
203 | |
204 | return 0; |
205 | } |
206 | |
207 | /* |
208 | * Generates Offsets and NextBurstLength based on Begrun and Runlength |
209 | * carried in a Data SNACK or ExpDataSN in TMR TASK_REASSIGN. |
210 | * |
211 | * For DataSequenceInOrder=No and DataPDUInOrder=[Yes,No] only. |
212 | * |
213 | * FIXME: How is this handled for a RData SNACK? |
214 | */ |
215 | int iscsit_create_recovery_datain_values_datasequenceinorder_no( |
216 | struct iscsit_cmd *cmd, |
217 | struct iscsi_datain_req *dr) |
218 | { |
219 | int found_seq = 0, i; |
220 | u32 data_sn, read_data_done = 0, seq_send_order = 0; |
221 | u32 begrun = dr->begrun; |
222 | u32 runlength = dr->runlength; |
223 | struct iscsit_conn *conn = cmd->conn; |
224 | struct iscsi_seq *first_seq = NULL, *seq = NULL; |
225 | |
226 | if (!cmd->seq_list) { |
227 | pr_err("struct iscsit_cmd->seq_list is NULL!\n" ); |
228 | return -1; |
229 | } |
230 | |
231 | /* |
232 | * Calculate read_data_done for all sequences containing a |
233 | * first_datasn and last_datasn less than the BegRun. |
234 | * |
235 | * Locate the struct iscsi_seq the BegRun lies within and calculate |
236 | * NextBurstLenghth up to the DataSN based on MaxRecvDataSegmentLength. |
237 | * |
238 | * Also use struct iscsi_seq->seq_send_order to determine where to start. |
239 | */ |
240 | for (i = 0; i < cmd->seq_count; i++) { |
241 | seq = &cmd->seq_list[i]; |
242 | |
243 | if (!seq->seq_send_order) |
244 | first_seq = seq; |
245 | |
246 | /* |
247 | * No data has been transferred for this DataIN sequence, so the |
248 | * seq->first_datasn and seq->last_datasn have not been set. |
249 | */ |
250 | if (!seq->sent) { |
251 | pr_err("Ignoring non-sent sequence 0x%08x ->" |
252 | " 0x%08x\n\n" , seq->first_datasn, |
253 | seq->last_datasn); |
254 | continue; |
255 | } |
256 | |
257 | /* |
258 | * This DataIN sequence is precedes the received BegRun, add the |
259 | * total xfer_len of the sequence to read_data_done and reset |
260 | * seq->pdu_send_order. |
261 | */ |
262 | if ((seq->first_datasn < begrun) && |
263 | (seq->last_datasn < begrun)) { |
264 | pr_err("Pre BegRun sequence 0x%08x ->" |
265 | " 0x%08x\n" , seq->first_datasn, |
266 | seq->last_datasn); |
267 | |
268 | read_data_done += cmd->seq_list[i].xfer_len; |
269 | seq->next_burst_len = seq->pdu_send_order = 0; |
270 | continue; |
271 | } |
272 | |
273 | /* |
274 | * The BegRun lies within this DataIN sequence. |
275 | */ |
276 | if ((seq->first_datasn <= begrun) && |
277 | (seq->last_datasn >= begrun)) { |
278 | pr_err("Found sequence begrun: 0x%08x in" |
279 | " 0x%08x -> 0x%08x\n" , begrun, |
280 | seq->first_datasn, seq->last_datasn); |
281 | |
282 | seq_send_order = seq->seq_send_order; |
283 | data_sn = seq->first_datasn; |
284 | seq->next_burst_len = seq->pdu_send_order = 0; |
285 | found_seq = 1; |
286 | |
287 | /* |
288 | * For DataPDUInOrder=Yes, while the first DataSN of |
289 | * the sequence is less than the received BegRun, add |
290 | * the MaxRecvDataSegmentLength to read_data_done and |
291 | * to the sequence's next_burst_len; |
292 | * |
293 | * For DataPDUInOrder=No, while the first DataSN of the |
294 | * sequence is less than the received BegRun, find the |
295 | * struct iscsi_pdu of the DataSN in question and add the |
296 | * MaxRecvDataSegmentLength to read_data_done and to the |
297 | * sequence's next_burst_len; |
298 | */ |
299 | if (conn->sess->sess_ops->DataPDUInOrder) { |
300 | while (data_sn < begrun) { |
301 | seq->pdu_send_order++; |
302 | read_data_done += |
303 | conn->conn_ops->MaxRecvDataSegmentLength; |
304 | seq->next_burst_len += |
305 | conn->conn_ops->MaxRecvDataSegmentLength; |
306 | data_sn++; |
307 | } |
308 | } else { |
309 | int j; |
310 | struct iscsi_pdu *pdu; |
311 | |
312 | while (data_sn < begrun) { |
313 | seq->pdu_send_order++; |
314 | |
315 | for (j = 0; j < seq->pdu_count; j++) { |
316 | pdu = &cmd->pdu_list[ |
317 | seq->pdu_start + j]; |
318 | if (pdu->data_sn == data_sn) { |
319 | read_data_done += |
320 | pdu->length; |
321 | seq->next_burst_len += |
322 | pdu->length; |
323 | } |
324 | } |
325 | data_sn++; |
326 | } |
327 | } |
328 | continue; |
329 | } |
330 | |
331 | /* |
332 | * This DataIN sequence is larger than the received BegRun, |
333 | * reset seq->pdu_send_order and continue. |
334 | */ |
335 | if ((seq->first_datasn > begrun) || |
336 | (seq->last_datasn > begrun)) { |
337 | pr_err("Post BegRun sequence 0x%08x -> 0x%08x\n" , |
338 | seq->first_datasn, seq->last_datasn); |
339 | |
340 | seq->next_burst_len = seq->pdu_send_order = 0; |
341 | continue; |
342 | } |
343 | } |
344 | |
345 | if (!found_seq) { |
346 | if (!begrun) { |
347 | if (!first_seq) { |
348 | pr_err("ITT: 0x%08x, Begrun: 0x%08x" |
349 | " but first_seq is NULL\n" , |
350 | cmd->init_task_tag, begrun); |
351 | return -1; |
352 | } |
353 | seq_send_order = first_seq->seq_send_order; |
354 | seq->next_burst_len = seq->pdu_send_order = 0; |
355 | goto done; |
356 | } |
357 | |
358 | pr_err("Unable to locate struct iscsi_seq for ITT: 0x%08x," |
359 | " BegRun: 0x%08x, RunLength: 0x%08x while" |
360 | " DataSequenceInOrder=No and DataPDUInOrder=%s.\n" , |
361 | cmd->init_task_tag, begrun, runlength, |
362 | (conn->sess->sess_ops->DataPDUInOrder) ? "Yes" : "No" ); |
363 | return -1; |
364 | } |
365 | |
366 | done: |
367 | dr->read_data_done = read_data_done; |
368 | dr->seq_send_order = seq_send_order; |
369 | |
370 | return 0; |
371 | } |
372 | |
373 | static int iscsit_handle_recovery_datain( |
374 | struct iscsit_cmd *cmd, |
375 | unsigned char *buf, |
376 | u32 begrun, |
377 | u32 runlength) |
378 | { |
379 | struct iscsit_conn *conn = cmd->conn; |
380 | struct iscsi_datain_req *dr; |
381 | struct se_cmd *se_cmd = &cmd->se_cmd; |
382 | |
383 | if (!(se_cmd->transport_state & CMD_T_COMPLETE)) { |
384 | pr_err("Ignoring ITT: 0x%08x Data SNACK\n" , |
385 | cmd->init_task_tag); |
386 | return 0; |
387 | } |
388 | |
389 | /* |
390 | * Make sure the initiator is not requesting retransmission |
391 | * of DataSNs already acknowledged by a Data ACK SNACK. |
392 | */ |
393 | if ((cmd->cmd_flags & ICF_GOT_DATACK_SNACK) && |
394 | (begrun <= cmd->acked_data_sn)) { |
395 | pr_err("ITT: 0x%08x, Data SNACK requesting" |
396 | " retransmission of DataSN: 0x%08x to 0x%08x but" |
397 | " already acked to DataSN: 0x%08x by Data ACK SNACK," |
398 | " protocol error.\n" , cmd->init_task_tag, begrun, |
399 | (begrun + runlength), cmd->acked_data_sn); |
400 | |
401 | return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf); |
402 | } |
403 | |
404 | /* |
405 | * Make sure BegRun and RunLength in the Data SNACK are sane. |
406 | * Note: (cmd->data_sn - 1) will carry the maximum DataSN sent. |
407 | */ |
408 | if ((begrun + runlength) > (cmd->data_sn - 1)) { |
409 | pr_err("Initiator requesting BegRun: 0x%08x, RunLength" |
410 | ": 0x%08x greater than maximum DataSN: 0x%08x.\n" , |
411 | begrun, runlength, (cmd->data_sn - 1)); |
412 | return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, |
413 | buf); |
414 | } |
415 | |
416 | dr = iscsit_allocate_datain_req(); |
417 | if (!dr) |
418 | return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_NO_RESOURCES, |
419 | buf); |
420 | |
421 | dr->data_sn = dr->begrun = begrun; |
422 | dr->runlength = runlength; |
423 | dr->generate_recovery_values = 1; |
424 | dr->recovery = DATAIN_WITHIN_COMMAND_RECOVERY; |
425 | |
426 | iscsit_attach_datain_req(cmd, dr); |
427 | |
428 | cmd->i_state = ISTATE_SEND_DATAIN; |
429 | iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); |
430 | |
431 | return 0; |
432 | } |
433 | |
434 | int iscsit_handle_recovery_datain_or_r2t( |
435 | struct iscsit_conn *conn, |
436 | unsigned char *buf, |
437 | itt_t init_task_tag, |
438 | u32 targ_xfer_tag, |
439 | u32 begrun, |
440 | u32 runlength) |
441 | { |
442 | struct iscsit_cmd *cmd; |
443 | |
444 | cmd = iscsit_find_cmd_from_itt(conn, init_task_tag); |
445 | if (!cmd) |
446 | return 0; |
447 | |
448 | /* |
449 | * FIXME: This will not work for bidi commands. |
450 | */ |
451 | switch (cmd->data_direction) { |
452 | case DMA_TO_DEVICE: |
453 | return iscsit_handle_r2t_snack(cmd, buf, begrun, runlength); |
454 | case DMA_FROM_DEVICE: |
455 | return iscsit_handle_recovery_datain(cmd, buf, begrun, |
456 | runlength); |
457 | default: |
458 | pr_err("Unknown cmd->data_direction: 0x%02x\n" , |
459 | cmd->data_direction); |
460 | return -1; |
461 | } |
462 | |
463 | return 0; |
464 | } |
465 | |
466 | /* #warning FIXME: Status SNACK needs to be dependent on OPCODE!!! */ |
467 | int iscsit_handle_status_snack( |
468 | struct iscsit_conn *conn, |
469 | itt_t init_task_tag, |
470 | u32 targ_xfer_tag, |
471 | u32 begrun, |
472 | u32 runlength) |
473 | { |
474 | struct iscsit_cmd *cmd = NULL; |
475 | u32 last_statsn; |
476 | int found_cmd; |
477 | |
478 | if (!begrun) { |
479 | begrun = conn->exp_statsn; |
480 | } else if (conn->exp_statsn > begrun) { |
481 | pr_err("Got Status SNACK Begrun: 0x%08x, RunLength:" |
482 | " 0x%08x but already got ExpStatSN: 0x%08x on CID:" |
483 | " %hu.\n" , begrun, runlength, conn->exp_statsn, |
484 | conn->cid); |
485 | return 0; |
486 | } |
487 | |
488 | last_statsn = (!runlength) ? conn->stat_sn : (begrun + runlength); |
489 | |
490 | while (begrun < last_statsn) { |
491 | found_cmd = 0; |
492 | |
493 | spin_lock_bh(lock: &conn->cmd_lock); |
494 | list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) { |
495 | if (cmd->stat_sn == begrun) { |
496 | found_cmd = 1; |
497 | break; |
498 | } |
499 | } |
500 | spin_unlock_bh(lock: &conn->cmd_lock); |
501 | |
502 | if (!found_cmd) { |
503 | pr_err("Unable to find StatSN: 0x%08x for" |
504 | " a Status SNACK, assuming this was a" |
505 | " protactic SNACK for an untransmitted" |
506 | " StatSN, ignoring.\n" , begrun); |
507 | begrun++; |
508 | continue; |
509 | } |
510 | |
511 | spin_lock_bh(lock: &cmd->istate_lock); |
512 | if (cmd->i_state == ISTATE_SEND_DATAIN) { |
513 | spin_unlock_bh(lock: &cmd->istate_lock); |
514 | pr_err("Ignoring Status SNACK for BegRun:" |
515 | " 0x%08x, RunLength: 0x%08x, assuming this was" |
516 | " a protactic SNACK for an untransmitted" |
517 | " StatSN\n" , begrun, runlength); |
518 | begrun++; |
519 | continue; |
520 | } |
521 | spin_unlock_bh(lock: &cmd->istate_lock); |
522 | |
523 | cmd->i_state = ISTATE_SEND_STATUS_RECOVERY; |
524 | iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); |
525 | begrun++; |
526 | } |
527 | |
528 | return 0; |
529 | } |
530 | |
531 | int iscsit_handle_data_ack( |
532 | struct iscsit_conn *conn, |
533 | u32 targ_xfer_tag, |
534 | u32 begrun, |
535 | u32 runlength) |
536 | { |
537 | struct iscsit_cmd *cmd = NULL; |
538 | |
539 | cmd = iscsit_find_cmd_from_ttt(conn, targ_xfer_tag); |
540 | if (!cmd) { |
541 | pr_err("Data ACK SNACK for TTT: 0x%08x is" |
542 | " invalid.\n" , targ_xfer_tag); |
543 | return -1; |
544 | } |
545 | |
546 | if (begrun <= cmd->acked_data_sn) { |
547 | pr_err("ITT: 0x%08x Data ACK SNACK BegRUN: 0x%08x is" |
548 | " less than the already acked DataSN: 0x%08x.\n" , |
549 | cmd->init_task_tag, begrun, cmd->acked_data_sn); |
550 | return -1; |
551 | } |
552 | |
553 | /* |
554 | * For Data ACK SNACK, BegRun is the next expected DataSN. |
555 | * (see iSCSI v19: 10.16.6) |
556 | */ |
557 | cmd->cmd_flags |= ICF_GOT_DATACK_SNACK; |
558 | cmd->acked_data_sn = (begrun - 1); |
559 | |
560 | pr_debug("Received Data ACK SNACK for ITT: 0x%08x," |
561 | " updated acked DataSN to 0x%08x.\n" , |
562 | cmd->init_task_tag, cmd->acked_data_sn); |
563 | |
564 | return 0; |
565 | } |
566 | |
567 | static int iscsit_send_recovery_r2t( |
568 | struct iscsit_cmd *cmd, |
569 | u32 offset, |
570 | u32 xfer_len) |
571 | { |
572 | int ret; |
573 | |
574 | spin_lock_bh(lock: &cmd->r2t_lock); |
575 | ret = iscsit_add_r2t_to_list(cmd, offset, xfer_len, 1, 0); |
576 | spin_unlock_bh(lock: &cmd->r2t_lock); |
577 | |
578 | return ret; |
579 | } |
580 | |
581 | int iscsit_dataout_datapduinorder_no_fbit( |
582 | struct iscsit_cmd *cmd, |
583 | struct iscsi_pdu *pdu) |
584 | { |
585 | int i, send_recovery_r2t = 0, recovery = 0; |
586 | u32 length = 0, offset = 0, pdu_count = 0, xfer_len = 0; |
587 | struct iscsit_conn *conn = cmd->conn; |
588 | struct iscsi_pdu *first_pdu = NULL; |
589 | |
590 | /* |
591 | * Get an struct iscsi_pdu pointer to the first PDU, and total PDU count |
592 | * of the DataOUT sequence. |
593 | */ |
594 | if (conn->sess->sess_ops->DataSequenceInOrder) { |
595 | for (i = 0; i < cmd->pdu_count; i++) { |
596 | if (cmd->pdu_list[i].seq_no == pdu->seq_no) { |
597 | if (!first_pdu) |
598 | first_pdu = &cmd->pdu_list[i]; |
599 | xfer_len += cmd->pdu_list[i].length; |
600 | pdu_count++; |
601 | } else if (pdu_count) |
602 | break; |
603 | } |
604 | } else { |
605 | struct iscsi_seq *seq = cmd->seq_ptr; |
606 | |
607 | first_pdu = &cmd->pdu_list[seq->pdu_start]; |
608 | pdu_count = seq->pdu_count; |
609 | } |
610 | |
611 | if (!first_pdu || !pdu_count) |
612 | return DATAOUT_CANNOT_RECOVER; |
613 | |
614 | /* |
615 | * Loop through the ending DataOUT Sequence checking each struct iscsi_pdu. |
616 | * The following ugly logic does batching of not received PDUs. |
617 | */ |
618 | for (i = 0; i < pdu_count; i++) { |
619 | if (first_pdu[i].status == ISCSI_PDU_RECEIVED_OK) { |
620 | if (!send_recovery_r2t) |
621 | continue; |
622 | |
623 | if (iscsit_send_recovery_r2t(cmd, offset, xfer_len: length) < 0) |
624 | return DATAOUT_CANNOT_RECOVER; |
625 | |
626 | send_recovery_r2t = length = offset = 0; |
627 | continue; |
628 | } |
629 | /* |
630 | * Set recovery = 1 for any missing, CRC failed, or timed |
631 | * out PDUs to let the DataOUT logic know that this sequence |
632 | * has not been completed yet. |
633 | * |
634 | * Also, only send a Recovery R2T for ISCSI_PDU_NOT_RECEIVED. |
635 | * We assume if the PDU either failed CRC or timed out |
636 | * that a Recovery R2T has already been sent. |
637 | */ |
638 | recovery = 1; |
639 | |
640 | if (first_pdu[i].status != ISCSI_PDU_NOT_RECEIVED) |
641 | continue; |
642 | |
643 | if (!offset) |
644 | offset = first_pdu[i].offset; |
645 | length += first_pdu[i].length; |
646 | |
647 | send_recovery_r2t = 1; |
648 | } |
649 | |
650 | if (send_recovery_r2t) |
651 | if (iscsit_send_recovery_r2t(cmd, offset, xfer_len: length) < 0) |
652 | return DATAOUT_CANNOT_RECOVER; |
653 | |
654 | return (!recovery) ? DATAOUT_NORMAL : DATAOUT_WITHIN_COMMAND_RECOVERY; |
655 | } |
656 | |
657 | static int iscsit_recalculate_dataout_values( |
658 | struct iscsit_cmd *cmd, |
659 | u32 pdu_offset, |
660 | u32 pdu_length, |
661 | u32 *r2t_offset, |
662 | u32 *r2t_length) |
663 | { |
664 | int i; |
665 | struct iscsit_conn *conn = cmd->conn; |
666 | struct iscsi_pdu *pdu = NULL; |
667 | |
668 | if (conn->sess->sess_ops->DataSequenceInOrder) { |
669 | cmd->data_sn = 0; |
670 | |
671 | if (conn->sess->sess_ops->DataPDUInOrder) { |
672 | *r2t_offset = cmd->write_data_done; |
673 | *r2t_length = (cmd->seq_end_offset - |
674 | cmd->write_data_done); |
675 | return 0; |
676 | } |
677 | |
678 | *r2t_offset = cmd->seq_start_offset; |
679 | *r2t_length = (cmd->seq_end_offset - cmd->seq_start_offset); |
680 | |
681 | for (i = 0; i < cmd->pdu_count; i++) { |
682 | pdu = &cmd->pdu_list[i]; |
683 | |
684 | if (pdu->status != ISCSI_PDU_RECEIVED_OK) |
685 | continue; |
686 | |
687 | if ((pdu->offset >= cmd->seq_start_offset) && |
688 | ((pdu->offset + pdu->length) <= |
689 | cmd->seq_end_offset)) { |
690 | if (!cmd->unsolicited_data) |
691 | cmd->next_burst_len -= pdu->length; |
692 | else |
693 | cmd->first_burst_len -= pdu->length; |
694 | |
695 | cmd->write_data_done -= pdu->length; |
696 | pdu->status = ISCSI_PDU_NOT_RECEIVED; |
697 | } |
698 | } |
699 | } else { |
700 | struct iscsi_seq *seq = NULL; |
701 | |
702 | seq = iscsit_get_seq_holder(cmd, pdu_offset, pdu_length); |
703 | if (!seq) |
704 | return -1; |
705 | |
706 | *r2t_offset = seq->orig_offset; |
707 | *r2t_length = seq->xfer_len; |
708 | |
709 | cmd->write_data_done -= (seq->offset - seq->orig_offset); |
710 | if (cmd->immediate_data) |
711 | cmd->first_burst_len = cmd->write_data_done; |
712 | |
713 | seq->data_sn = 0; |
714 | seq->offset = seq->orig_offset; |
715 | seq->next_burst_len = 0; |
716 | seq->status = DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY; |
717 | |
718 | if (conn->sess->sess_ops->DataPDUInOrder) |
719 | return 0; |
720 | |
721 | for (i = 0; i < seq->pdu_count; i++) { |
722 | pdu = &cmd->pdu_list[i+seq->pdu_start]; |
723 | |
724 | if (pdu->status != ISCSI_PDU_RECEIVED_OK) |
725 | continue; |
726 | |
727 | pdu->status = ISCSI_PDU_NOT_RECEIVED; |
728 | } |
729 | } |
730 | |
731 | return 0; |
732 | } |
733 | |
734 | int iscsit_recover_dataout_sequence( |
735 | struct iscsit_cmd *cmd, |
736 | u32 pdu_offset, |
737 | u32 pdu_length) |
738 | { |
739 | u32 r2t_length = 0, r2t_offset = 0; |
740 | |
741 | spin_lock_bh(lock: &cmd->istate_lock); |
742 | cmd->cmd_flags |= ICF_WITHIN_COMMAND_RECOVERY; |
743 | spin_unlock_bh(lock: &cmd->istate_lock); |
744 | |
745 | if (iscsit_recalculate_dataout_values(cmd, pdu_offset, pdu_length, |
746 | r2t_offset: &r2t_offset, r2t_length: &r2t_length) < 0) |
747 | return DATAOUT_CANNOT_RECOVER; |
748 | |
749 | iscsit_send_recovery_r2t(cmd, offset: r2t_offset, xfer_len: r2t_length); |
750 | |
751 | return DATAOUT_WITHIN_COMMAND_RECOVERY; |
752 | } |
753 | |
754 | static struct iscsi_ooo_cmdsn *iscsit_allocate_ooo_cmdsn(void) |
755 | { |
756 | struct iscsi_ooo_cmdsn *ooo_cmdsn = NULL; |
757 | |
758 | ooo_cmdsn = kmem_cache_zalloc(k: lio_ooo_cache, GFP_ATOMIC); |
759 | if (!ooo_cmdsn) { |
760 | pr_err("Unable to allocate memory for" |
761 | " struct iscsi_ooo_cmdsn.\n" ); |
762 | return NULL; |
763 | } |
764 | INIT_LIST_HEAD(list: &ooo_cmdsn->ooo_list); |
765 | |
766 | return ooo_cmdsn; |
767 | } |
768 | |
769 | static int iscsit_attach_ooo_cmdsn( |
770 | struct iscsit_session *sess, |
771 | struct iscsi_ooo_cmdsn *ooo_cmdsn) |
772 | { |
773 | struct iscsi_ooo_cmdsn *ooo_tail, *ooo_tmp; |
774 | |
775 | lockdep_assert_held(&sess->cmdsn_mutex); |
776 | |
777 | /* |
778 | * We attach the struct iscsi_ooo_cmdsn entry to the out of order |
779 | * list in increasing CmdSN order. |
780 | * This allows iscsi_execute_ooo_cmdsns() to detect any |
781 | * additional CmdSN holes while performing delayed execution. |
782 | */ |
783 | if (list_empty(head: &sess->sess_ooo_cmdsn_list)) |
784 | list_add_tail(new: &ooo_cmdsn->ooo_list, |
785 | head: &sess->sess_ooo_cmdsn_list); |
786 | else { |
787 | ooo_tail = list_entry(sess->sess_ooo_cmdsn_list.prev, |
788 | typeof(*ooo_tail), ooo_list); |
789 | /* |
790 | * CmdSN is greater than the tail of the list. |
791 | */ |
792 | if (iscsi_sna_lt(n1: ooo_tail->cmdsn, n2: ooo_cmdsn->cmdsn)) |
793 | list_add_tail(new: &ooo_cmdsn->ooo_list, |
794 | head: &sess->sess_ooo_cmdsn_list); |
795 | else { |
796 | /* |
797 | * CmdSN is either lower than the head, or somewhere |
798 | * in the middle. |
799 | */ |
800 | list_for_each_entry(ooo_tmp, &sess->sess_ooo_cmdsn_list, |
801 | ooo_list) { |
802 | if (iscsi_sna_lt(n1: ooo_tmp->cmdsn, n2: ooo_cmdsn->cmdsn)) |
803 | continue; |
804 | |
805 | /* Insert before this entry */ |
806 | list_add(new: &ooo_cmdsn->ooo_list, |
807 | head: ooo_tmp->ooo_list.prev); |
808 | break; |
809 | } |
810 | } |
811 | } |
812 | |
813 | return 0; |
814 | } |
815 | |
816 | /* |
817 | * Removes an struct iscsi_ooo_cmdsn from a session's list, |
818 | * called with struct iscsit_session->cmdsn_mutex held. |
819 | */ |
820 | void iscsit_remove_ooo_cmdsn( |
821 | struct iscsit_session *sess, |
822 | struct iscsi_ooo_cmdsn *ooo_cmdsn) |
823 | { |
824 | list_del(entry: &ooo_cmdsn->ooo_list); |
825 | kmem_cache_free(s: lio_ooo_cache, objp: ooo_cmdsn); |
826 | } |
827 | |
828 | void iscsit_clear_ooo_cmdsns_for_conn(struct iscsit_conn *conn) |
829 | { |
830 | struct iscsi_ooo_cmdsn *ooo_cmdsn; |
831 | struct iscsit_session *sess = conn->sess; |
832 | |
833 | mutex_lock(&sess->cmdsn_mutex); |
834 | list_for_each_entry(ooo_cmdsn, &sess->sess_ooo_cmdsn_list, ooo_list) { |
835 | if (ooo_cmdsn->cid != conn->cid) |
836 | continue; |
837 | |
838 | ooo_cmdsn->cmd = NULL; |
839 | } |
840 | mutex_unlock(lock: &sess->cmdsn_mutex); |
841 | } |
842 | |
843 | int iscsit_execute_ooo_cmdsns(struct iscsit_session *sess) |
844 | { |
845 | int ooo_count = 0; |
846 | struct iscsit_cmd *cmd = NULL; |
847 | struct iscsi_ooo_cmdsn *ooo_cmdsn, *ooo_cmdsn_tmp; |
848 | |
849 | lockdep_assert_held(&sess->cmdsn_mutex); |
850 | |
851 | list_for_each_entry_safe(ooo_cmdsn, ooo_cmdsn_tmp, |
852 | &sess->sess_ooo_cmdsn_list, ooo_list) { |
853 | if (ooo_cmdsn->cmdsn != sess->exp_cmd_sn) |
854 | continue; |
855 | |
856 | if (!ooo_cmdsn->cmd) { |
857 | sess->exp_cmd_sn++; |
858 | iscsit_remove_ooo_cmdsn(sess, ooo_cmdsn); |
859 | continue; |
860 | } |
861 | |
862 | cmd = ooo_cmdsn->cmd; |
863 | cmd->i_state = cmd->deferred_i_state; |
864 | ooo_count++; |
865 | sess->exp_cmd_sn++; |
866 | pr_debug("Executing out of order CmdSN: 0x%08x," |
867 | " incremented ExpCmdSN to 0x%08x.\n" , |
868 | cmd->cmd_sn, sess->exp_cmd_sn); |
869 | |
870 | iscsit_remove_ooo_cmdsn(sess, ooo_cmdsn); |
871 | |
872 | if (iscsit_execute_cmd(cmd, 1) < 0) |
873 | return -1; |
874 | } |
875 | |
876 | return ooo_count; |
877 | } |
878 | |
879 | /* |
880 | * Called either: |
881 | * |
882 | * 1. With sess->cmdsn_mutex held from iscsi_execute_ooo_cmdsns() |
883 | * or iscsi_check_received_cmdsn(). |
884 | * 2. With no locks held directly from iscsi_handle_XXX_pdu() functions |
885 | * for immediate commands. |
886 | */ |
887 | int iscsit_execute_cmd(struct iscsit_cmd *cmd, int ooo) |
888 | { |
889 | struct se_cmd *se_cmd = &cmd->se_cmd; |
890 | struct iscsit_conn *conn = cmd->conn; |
891 | int lr = 0; |
892 | |
893 | spin_lock_bh(lock: &cmd->istate_lock); |
894 | if (ooo) |
895 | cmd->cmd_flags &= ~ICF_OOO_CMDSN; |
896 | |
897 | switch (cmd->iscsi_opcode) { |
898 | case ISCSI_OP_SCSI_CMD: |
899 | /* |
900 | * Go ahead and send the CHECK_CONDITION status for |
901 | * any SCSI CDB exceptions that may have occurred. |
902 | */ |
903 | if (cmd->sense_reason) { |
904 | if (cmd->sense_reason == TCM_RESERVATION_CONFLICT) { |
905 | cmd->i_state = ISTATE_SEND_STATUS; |
906 | spin_unlock_bh(lock: &cmd->istate_lock); |
907 | iscsit_add_cmd_to_response_queue(cmd, cmd->conn, |
908 | cmd->i_state); |
909 | return 0; |
910 | } |
911 | spin_unlock_bh(lock: &cmd->istate_lock); |
912 | if (cmd->se_cmd.transport_state & CMD_T_ABORTED) |
913 | return 0; |
914 | return transport_send_check_condition_and_sense(se_cmd, |
915 | cmd->sense_reason, 0); |
916 | } |
917 | /* |
918 | * Special case for delayed CmdSN with Immediate |
919 | * Data and/or Unsolicited Data Out attached. |
920 | */ |
921 | if (cmd->immediate_data) { |
922 | if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) { |
923 | spin_unlock_bh(lock: &cmd->istate_lock); |
924 | target_execute_cmd(cmd: &cmd->se_cmd); |
925 | return 0; |
926 | } |
927 | spin_unlock_bh(lock: &cmd->istate_lock); |
928 | |
929 | if (!(cmd->cmd_flags & |
930 | ICF_NON_IMMEDIATE_UNSOLICITED_DATA)) { |
931 | if (cmd->se_cmd.transport_state & CMD_T_ABORTED) |
932 | return 0; |
933 | |
934 | iscsit_set_dataout_sequence_values(cmd); |
935 | conn->conn_transport->iscsit_get_dataout(conn, cmd, false); |
936 | } |
937 | return 0; |
938 | } |
939 | /* |
940 | * The default handler. |
941 | */ |
942 | spin_unlock_bh(lock: &cmd->istate_lock); |
943 | |
944 | if ((cmd->data_direction == DMA_TO_DEVICE) && |
945 | !(cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA)) { |
946 | if (cmd->se_cmd.transport_state & CMD_T_ABORTED) |
947 | return 0; |
948 | |
949 | iscsit_set_unsolicited_dataout(cmd); |
950 | } |
951 | return target_submit(se_cmd: &cmd->se_cmd); |
952 | |
953 | case ISCSI_OP_NOOP_OUT: |
954 | case ISCSI_OP_TEXT: |
955 | spin_unlock_bh(lock: &cmd->istate_lock); |
956 | iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); |
957 | break; |
958 | case ISCSI_OP_SCSI_TMFUNC: |
959 | if (cmd->se_cmd.se_tmr_req->response) { |
960 | spin_unlock_bh(lock: &cmd->istate_lock); |
961 | iscsit_add_cmd_to_response_queue(cmd, cmd->conn, |
962 | cmd->i_state); |
963 | return 0; |
964 | } |
965 | spin_unlock_bh(lock: &cmd->istate_lock); |
966 | |
967 | return transport_generic_handle_tmr(&cmd->se_cmd); |
968 | case ISCSI_OP_LOGOUT: |
969 | spin_unlock_bh(lock: &cmd->istate_lock); |
970 | switch (cmd->logout_reason) { |
971 | case ISCSI_LOGOUT_REASON_CLOSE_SESSION: |
972 | lr = iscsit_logout_closesession(cmd, cmd->conn); |
973 | break; |
974 | case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: |
975 | lr = iscsit_logout_closeconnection(cmd, cmd->conn); |
976 | break; |
977 | case ISCSI_LOGOUT_REASON_RECOVERY: |
978 | lr = iscsit_logout_removeconnforrecovery(cmd, cmd->conn); |
979 | break; |
980 | default: |
981 | pr_err("Unknown iSCSI Logout Request Code:" |
982 | " 0x%02x\n" , cmd->logout_reason); |
983 | return -1; |
984 | } |
985 | |
986 | return lr; |
987 | default: |
988 | spin_unlock_bh(lock: &cmd->istate_lock); |
989 | pr_err("Cannot perform out of order execution for" |
990 | " unknown iSCSI Opcode: 0x%02x\n" , cmd->iscsi_opcode); |
991 | return -1; |
992 | } |
993 | |
994 | return 0; |
995 | } |
996 | |
997 | void iscsit_free_all_ooo_cmdsns(struct iscsit_session *sess) |
998 | { |
999 | struct iscsi_ooo_cmdsn *ooo_cmdsn, *ooo_cmdsn_tmp; |
1000 | |
1001 | mutex_lock(&sess->cmdsn_mutex); |
1002 | list_for_each_entry_safe(ooo_cmdsn, ooo_cmdsn_tmp, |
1003 | &sess->sess_ooo_cmdsn_list, ooo_list) { |
1004 | |
1005 | list_del(entry: &ooo_cmdsn->ooo_list); |
1006 | kmem_cache_free(s: lio_ooo_cache, objp: ooo_cmdsn); |
1007 | } |
1008 | mutex_unlock(lock: &sess->cmdsn_mutex); |
1009 | } |
1010 | |
1011 | int iscsit_handle_ooo_cmdsn( |
1012 | struct iscsit_session *sess, |
1013 | struct iscsit_cmd *cmd, |
1014 | u32 cmdsn) |
1015 | { |
1016 | int batch = 0; |
1017 | struct iscsi_ooo_cmdsn *ooo_cmdsn = NULL, *ooo_tail = NULL; |
1018 | |
1019 | cmd->deferred_i_state = cmd->i_state; |
1020 | cmd->i_state = ISTATE_DEFERRED_CMD; |
1021 | cmd->cmd_flags |= ICF_OOO_CMDSN; |
1022 | |
1023 | if (list_empty(head: &sess->sess_ooo_cmdsn_list)) |
1024 | batch = 1; |
1025 | else { |
1026 | ooo_tail = list_entry(sess->sess_ooo_cmdsn_list.prev, |
1027 | typeof(*ooo_tail), ooo_list); |
1028 | if (ooo_tail->cmdsn != (cmdsn - 1)) |
1029 | batch = 1; |
1030 | } |
1031 | |
1032 | ooo_cmdsn = iscsit_allocate_ooo_cmdsn(); |
1033 | if (!ooo_cmdsn) |
1034 | return -ENOMEM; |
1035 | |
1036 | ooo_cmdsn->cmd = cmd; |
1037 | ooo_cmdsn->batch_count = (batch) ? |
1038 | (cmdsn - sess->exp_cmd_sn) : 1; |
1039 | ooo_cmdsn->cid = cmd->conn->cid; |
1040 | ooo_cmdsn->exp_cmdsn = sess->exp_cmd_sn; |
1041 | ooo_cmdsn->cmdsn = cmdsn; |
1042 | |
1043 | if (iscsit_attach_ooo_cmdsn(sess, ooo_cmdsn) < 0) { |
1044 | kmem_cache_free(s: lio_ooo_cache, objp: ooo_cmdsn); |
1045 | return -ENOMEM; |
1046 | } |
1047 | |
1048 | return 0; |
1049 | } |
1050 | |
1051 | static int iscsit_set_dataout_timeout_values( |
1052 | struct iscsit_cmd *cmd, |
1053 | u32 *offset, |
1054 | u32 *length) |
1055 | { |
1056 | struct iscsit_conn *conn = cmd->conn; |
1057 | struct iscsi_r2t *r2t; |
1058 | |
1059 | if (cmd->unsolicited_data) { |
1060 | *offset = 0; |
1061 | *length = (conn->sess->sess_ops->FirstBurstLength > |
1062 | cmd->se_cmd.data_length) ? |
1063 | cmd->se_cmd.data_length : |
1064 | conn->sess->sess_ops->FirstBurstLength; |
1065 | return 0; |
1066 | } |
1067 | |
1068 | spin_lock_bh(lock: &cmd->r2t_lock); |
1069 | if (list_empty(head: &cmd->cmd_r2t_list)) { |
1070 | pr_err("cmd->cmd_r2t_list is empty!\n" ); |
1071 | spin_unlock_bh(lock: &cmd->r2t_lock); |
1072 | return -1; |
1073 | } |
1074 | |
1075 | list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) { |
1076 | if (r2t->sent_r2t && !r2t->recovery_r2t && !r2t->seq_complete) { |
1077 | *offset = r2t->offset; |
1078 | *length = r2t->xfer_len; |
1079 | spin_unlock_bh(lock: &cmd->r2t_lock); |
1080 | return 0; |
1081 | } |
1082 | } |
1083 | spin_unlock_bh(lock: &cmd->r2t_lock); |
1084 | |
1085 | pr_err("Unable to locate any incomplete DataOUT" |
1086 | " sequences for ITT: 0x%08x.\n" , cmd->init_task_tag); |
1087 | |
1088 | return -1; |
1089 | } |
1090 | |
1091 | /* |
1092 | * NOTE: Called from interrupt (timer) context. |
1093 | */ |
1094 | void iscsit_handle_dataout_timeout(struct timer_list *t) |
1095 | { |
1096 | u32 pdu_length = 0, pdu_offset = 0; |
1097 | u32 r2t_length = 0, r2t_offset = 0; |
1098 | struct iscsit_cmd *cmd = from_timer(cmd, t, dataout_timer); |
1099 | struct iscsit_conn *conn = cmd->conn; |
1100 | struct iscsit_session *sess = NULL; |
1101 | struct iscsi_node_attrib *na; |
1102 | |
1103 | iscsit_inc_conn_usage_count(conn); |
1104 | |
1105 | spin_lock_bh(lock: &cmd->dataout_timeout_lock); |
1106 | if (cmd->dataout_timer_flags & ISCSI_TF_STOP) { |
1107 | spin_unlock_bh(lock: &cmd->dataout_timeout_lock); |
1108 | iscsit_dec_conn_usage_count(conn); |
1109 | return; |
1110 | } |
1111 | cmd->dataout_timer_flags &= ~ISCSI_TF_RUNNING; |
1112 | sess = conn->sess; |
1113 | na = iscsit_tpg_get_node_attrib(sess); |
1114 | |
1115 | if (!sess->sess_ops->ErrorRecoveryLevel) { |
1116 | pr_err("Unable to recover from DataOut timeout while" |
1117 | " in ERL=0, closing iSCSI connection for I_T Nexus" |
1118 | " %s,i,0x%6phN,%s,t,0x%02x\n" , |
1119 | sess->sess_ops->InitiatorName, sess->isid, |
1120 | sess->tpg->tpg_tiqn->tiqn, (u32)sess->tpg->tpgt); |
1121 | goto failure; |
1122 | } |
1123 | |
1124 | if (++cmd->dataout_timeout_retries == na->dataout_timeout_retries) { |
1125 | pr_err("Command ITT: 0x%08x exceeded max retries" |
1126 | " for DataOUT timeout %u, closing iSCSI connection for" |
1127 | " I_T Nexus %s,i,0x%6phN,%s,t,0x%02x\n" , |
1128 | cmd->init_task_tag, na->dataout_timeout_retries, |
1129 | sess->sess_ops->InitiatorName, sess->isid, |
1130 | sess->tpg->tpg_tiqn->tiqn, (u32)sess->tpg->tpgt); |
1131 | goto failure; |
1132 | } |
1133 | |
1134 | cmd->cmd_flags |= ICF_WITHIN_COMMAND_RECOVERY; |
1135 | |
1136 | if (conn->sess->sess_ops->DataSequenceInOrder) { |
1137 | if (conn->sess->sess_ops->DataPDUInOrder) { |
1138 | pdu_offset = cmd->write_data_done; |
1139 | if ((pdu_offset + (conn->sess->sess_ops->MaxBurstLength - |
1140 | cmd->next_burst_len)) > cmd->se_cmd.data_length) |
1141 | pdu_length = (cmd->se_cmd.data_length - |
1142 | cmd->write_data_done); |
1143 | else |
1144 | pdu_length = (conn->sess->sess_ops->MaxBurstLength - |
1145 | cmd->next_burst_len); |
1146 | } else { |
1147 | pdu_offset = cmd->seq_start_offset; |
1148 | pdu_length = (cmd->seq_end_offset - |
1149 | cmd->seq_start_offset); |
1150 | } |
1151 | } else { |
1152 | if (iscsit_set_dataout_timeout_values(cmd, offset: &pdu_offset, |
1153 | length: &pdu_length) < 0) |
1154 | goto failure; |
1155 | } |
1156 | |
1157 | if (iscsit_recalculate_dataout_values(cmd, pdu_offset, pdu_length, |
1158 | r2t_offset: &r2t_offset, r2t_length: &r2t_length) < 0) |
1159 | goto failure; |
1160 | |
1161 | pr_debug("Command ITT: 0x%08x timed out waiting for" |
1162 | " completion of %sDataOUT Sequence Offset: %u, Length: %u\n" , |
1163 | cmd->init_task_tag, (cmd->unsolicited_data) ? "Unsolicited " : |
1164 | "" , r2t_offset, r2t_length); |
1165 | |
1166 | if (iscsit_send_recovery_r2t(cmd, offset: r2t_offset, xfer_len: r2t_length) < 0) |
1167 | goto failure; |
1168 | |
1169 | iscsit_start_dataout_timer(cmd, conn); |
1170 | spin_unlock_bh(lock: &cmd->dataout_timeout_lock); |
1171 | iscsit_dec_conn_usage_count(conn); |
1172 | |
1173 | return; |
1174 | |
1175 | failure: |
1176 | spin_unlock_bh(lock: &cmd->dataout_timeout_lock); |
1177 | iscsit_fill_cxn_timeout_err_stats(sess); |
1178 | iscsit_cause_connection_reinstatement(conn, 0); |
1179 | iscsit_dec_conn_usage_count(conn); |
1180 | } |
1181 | |
1182 | void iscsit_mod_dataout_timer(struct iscsit_cmd *cmd) |
1183 | { |
1184 | struct iscsit_conn *conn = cmd->conn; |
1185 | struct iscsit_session *sess = conn->sess; |
1186 | struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess); |
1187 | |
1188 | spin_lock_bh(lock: &cmd->dataout_timeout_lock); |
1189 | if (!(cmd->dataout_timer_flags & ISCSI_TF_RUNNING)) { |
1190 | spin_unlock_bh(lock: &cmd->dataout_timeout_lock); |
1191 | return; |
1192 | } |
1193 | |
1194 | mod_timer(timer: &cmd->dataout_timer, |
1195 | expires: (get_jiffies_64() + na->dataout_timeout * HZ)); |
1196 | pr_debug("Updated DataOUT timer for ITT: 0x%08x" , |
1197 | cmd->init_task_tag); |
1198 | spin_unlock_bh(lock: &cmd->dataout_timeout_lock); |
1199 | } |
1200 | |
1201 | void iscsit_start_dataout_timer( |
1202 | struct iscsit_cmd *cmd, |
1203 | struct iscsit_conn *conn) |
1204 | { |
1205 | struct iscsit_session *sess = conn->sess; |
1206 | struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess); |
1207 | |
1208 | lockdep_assert_held(&cmd->dataout_timeout_lock); |
1209 | |
1210 | if (cmd->dataout_timer_flags & ISCSI_TF_RUNNING) |
1211 | return; |
1212 | |
1213 | pr_debug("Starting DataOUT timer for ITT: 0x%08x on" |
1214 | " CID: %hu.\n" , cmd->init_task_tag, conn->cid); |
1215 | |
1216 | cmd->dataout_timer_flags &= ~ISCSI_TF_STOP; |
1217 | cmd->dataout_timer_flags |= ISCSI_TF_RUNNING; |
1218 | mod_timer(timer: &cmd->dataout_timer, expires: jiffies + na->dataout_timeout * HZ); |
1219 | } |
1220 | |
1221 | void iscsit_stop_dataout_timer(struct iscsit_cmd *cmd) |
1222 | { |
1223 | spin_lock_bh(lock: &cmd->dataout_timeout_lock); |
1224 | if (!(cmd->dataout_timer_flags & ISCSI_TF_RUNNING)) { |
1225 | spin_unlock_bh(lock: &cmd->dataout_timeout_lock); |
1226 | return; |
1227 | } |
1228 | cmd->dataout_timer_flags |= ISCSI_TF_STOP; |
1229 | spin_unlock_bh(lock: &cmd->dataout_timeout_lock); |
1230 | |
1231 | del_timer_sync(timer: &cmd->dataout_timer); |
1232 | |
1233 | spin_lock_bh(lock: &cmd->dataout_timeout_lock); |
1234 | cmd->dataout_timer_flags &= ~ISCSI_TF_RUNNING; |
1235 | pr_debug("Stopped DataOUT Timer for ITT: 0x%08x\n" , |
1236 | cmd->init_task_tag); |
1237 | spin_unlock_bh(lock: &cmd->dataout_timeout_lock); |
1238 | } |
1239 | EXPORT_SYMBOL(iscsit_stop_dataout_timer); |
1240 | |