[Devel,RH7] scsi/eh: fix hang adding ehandler wakeups after decrementing host_busy

Submitted by Pavel Tikhomirov on Sept. 5, 2017, 9:43 a.m.

Details

Message ID 20170905094353.5727-1-ptikhomirov@virtuozzo.com
State New
Series "scsi/eh: fix hang adding ehandler wakeups after decrementing host_busy"
Headers show

Commit Message

Pavel Tikhomirov Sept. 5, 2017, 9:43 a.m.
We have a problem on several our nodes with scsi EH. Imagine such an
order of execution of two threads:

CPU1 scsi_eh_scmd_add		CPU2 scsi_host_queue_ready
/* shost->host_busy == 1 initialy */

				if (shost->shost_state == SHOST_RECOVERY)
					/* does not get here */
					return 0;

lock(shost->host_lock);
shost->shost_state = SHOST_RECOVERY;

				busy = shost->host_busy++;
				/* host->can_queue == 1 initialy, busy == 1 */
				lock(shost->host_lock) /* wait */

shost->host_failed++;
/* shost->host_busy == 2, shost->host_failed == 1 */
call scsi_eh_wakeup(shost) {
	if (host_busy == host_failed) {
		/* does not get here */
		wake_up_process(shost->ehandler)
	}
}
unlock(shost->host_lock)

				/* acquire lock */
				shost->host_busy--;

Finaly we do not wakeup scsi_error_handler and all other commands
comming will hang as we are in never ending recovery state as there
is no one left to wakeup handler.

So scsi disc in these host becomes unresponsive and all bio on node
hangs. (We trigger these problem when scsi cmnds to DVD drive timeout.)

Main idea of the fix is to try to do wake up every time we decrement
host_busy or increment host_faild(the latter is already OK).

Now the very *last* one of busy threads getting host_lock after
decrementing host_busy will see all write operations on host's
shost_state, host_busy and host_failed completed thanks to implied
memory barriers on spin_lock/unlock, so at the time of busy==failed
we will trigger wakeup in at least one thread. (Thats why putting
recovery and failed check under lock)

https://jira.sw.ru/browse/PSBM-69788

Signed-off-by: Pavel Tikhomirov <ptikhomirov@virtuozzo.com>
---
 drivers/scsi/scsi_lib.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

Patch hide | download patch | download mbox

diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 0f55949765c7..ec211185abd6 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -320,13 +320,13 @@  void scsi_device_unbusy(struct scsi_device *sdev)
 	if (starget->can_queue > 0)
 		atomic_dec(&starget->target_busy);
 
+	spin_lock_irqsave(shost->host_lock, flags);
 	if (unlikely(scsi_host_in_recovery(shost) &&
 		     (shost->host_failed || shost->host_eh_scheduled))) {
-		spin_lock_irqsave(shost->host_lock, flags);
 		scsi_debug_log_shost(SCSI_DEVICE_UNBUSY_CALLS_EH_WAKEUP, shost);
 		scsi_eh_wakeup(shost);
-		spin_unlock_irqrestore(shost->host_lock, flags);
 	}
+	spin_unlock_irqrestore(shost->host_lock, flags);
 
 	atomic_dec(&sdev->device_busy);
 }
@@ -1568,6 +1568,13 @@  static inline int scsi_host_queue_ready(struct request_queue *q,
 out_dec:
 	scsi_debug_log_sdev(SCSI_HOST_QUEUE_READY_DEC_HOST_BUSY, sdev);
 	atomic_dec(&shost->host_busy);
+
+	spin_lock_irq(shost->host_lock);
+	if (unlikely(scsi_host_in_recovery(shost) &&
+		     (shost->host_failed || shost->host_eh_scheduled)))
+		scsi_eh_wakeup(shost);
+	spin_unlock_irq(shost->host_lock);
+
 	return 0;
 }
 
@@ -1958,6 +1965,13 @@  static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
 out_dec_host_busy:
 	scsi_debug_log_sdev(SCSI_QUEUE_RQ_DEC_HOST_BUSY, sdev);
 	atomic_dec(&shost->host_busy);
+
+	spin_lock_irq(shost->host_lock);
+	if (unlikely(scsi_host_in_recovery(shost) &&
+		     (shost->host_failed || shost->host_eh_scheduled)))
+		scsi_eh_wakeup(shost);
+	spin_unlock_irq(shost->host_lock);
+
 out_dec_target_busy:
 	if (scsi_target(sdev)->can_queue > 0)
 		atomic_dec(&scsi_target(sdev)->target_busy);