linux-vybrid_public/drivers/scsi/libsas/sas_scsi_host.c

/*
 * Serial Attached SCSI (SAS) class SCSI Host glue.
 *
 * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
 * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
 *
 * This file is licensed under GPLv2.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
 * USA
 *
 */

#include <linux/kthread.h>

#include "sas_internal.h"

#include <scsi/scsi_host.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_tcq.h>
#include <scsi/scsi.h>
#include <scsi/scsi_eh.h>
#include <scsi/scsi_transport.h>
#include <scsi/scsi_transport_sas.h>
#include "../scsi_sas_internal.h"
#include "../scsi_transport_api.h"
#include "../scsi_priv.h"

#include <linux/err.h>
#include <linux/blkdev.h>
#include <linux/freezer.h>
#include <linux/scatterlist.h>
#include <linux/libata.h>

/* ---------- SCSI Host glue ---------- */

#define TO_SAS_TASK(_scsi_cmd)  ((void *)(_scsi_cmd)->host_scribble)
#define ASSIGN_SAS_TASK(_sc, _t) do { (_sc)->host_scribble = (void *) _t; } while (0)

static void sas_scsi_task_done(struct sas_task *task)
{
	struct task_status_struct *ts = &task->task_status;
	struct scsi_cmnd *sc = task->uldd_task;
	struct sas_ha_struct *sas_ha = SHOST_TO_SAS_HA(sc->device->host);
	unsigned ts_flags = task->task_state_flags;
	int hs = 0, stat = 0;

	if (unlikely(!sc)) {
		SAS_DPRINTK("task_done called with non existing SCSI cmnd!\n");
		list_del_init(&task->list);
		sas_free_task(task);
		return;
	}

	if (ts->resp == SAS_TASK_UNDELIVERED) {
		/* transport error */
		hs = DID_NO_CONNECT;
	} else { /* ts->resp == SAS_TASK_COMPLETE */
		/* task delivered, what happened afterwards? */
		switch (ts->stat) {
		case SAS_DEV_NO_RESPONSE:
		case SAS_INTERRUPTED:
		case SAS_PHY_DOWN:
		case SAS_NAK_R_ERR:
		case SAS_OPEN_TO:
			hs = DID_NO_CONNECT;
			break;
		case SAS_DATA_UNDERRUN:
			scsi_set_resid(sc, ts->residual);
			if (scsi_bufflen(sc) - scsi_get_resid(sc) < sc->underflow)
				hs = DID_ERROR;
			break;
		case SAS_DATA_OVERRUN:
			hs = DID_ERROR;
			break;
		case SAS_QUEUE_FULL:
			hs = DID_SOFT_ERROR; /* retry */
			break;
		case SAS_DEVICE_UNKNOWN:
			hs = DID_BAD_TARGET;
			break;
		case SAS_SG_ERR:
			hs = DID_PARITY;
			break;
		case SAS_OPEN_REJECT:
			if (ts->open_rej_reason == SAS_OREJ_RSVD_RETRY)
				hs = DID_SOFT_ERROR; /* retry */
			else
				hs = DID_ERROR;
			break;
		case SAS_PROTO_RESPONSE:
			SAS_DPRINTK("LLDD:%s sent SAS_PROTO_RESP for an SSP "
				    "task; please report this\n",
				    task->dev->port->ha->sas_ha_name);
			break;
		case SAS_ABORTED_TASK:
			hs = DID_ABORT;
			break;
		case SAM_CHECK_COND:
			memcpy(sc->sense_buffer, ts->buf,
			       max(SCSI_SENSE_BUFFERSIZE, ts->buf_valid_size));
			stat = SAM_CHECK_COND;
			break;
		default:
			stat = ts->stat;
			break;
		}
	}
	ASSIGN_SAS_TASK(sc, NULL);
	sc->result = (hs << 16) | stat;
	list_del_init(&task->list);
	sas_free_task(task);
	/* This is very ugly but this is how SCSI Core works. */
	if (ts_flags & SAS_TASK_STATE_ABORTED)
		scsi_eh_finish_cmd(sc, &sas_ha->eh_done_q);
	else
		sc->scsi_done(sc);
}

static enum task_attribute sas_scsi_get_task_attr(struct scsi_cmnd *cmd)
{
	enum task_attribute ta = TASK_ATTR_SIMPLE;
	if (cmd->request && blk_rq_tagged(cmd->request)) {
		if (cmd->device->ordered_tags &&
		    (cmd->request->cmd_flags & REQ_HARDBARRIER))
			ta = TASK_ATTR_ORDERED;
	}
	return ta;
}

static struct sas_task *sas_create_task(struct scsi_cmnd *cmd,
					       struct domain_device *dev,
					       gfp_t gfp_flags)
{
	struct sas_task *task = sas_alloc_task(gfp_flags);
	struct scsi_lun lun;

	if (!task)
		return NULL;

	*(u32 *)cmd->sense_buffer = 0;
	task->uldd_task = cmd;
	ASSIGN_SAS_TASK(cmd, task);

	task->dev = dev;
	task->task_proto = task->dev->tproto; /* BUG_ON(!SSP) */

	task->ssp_task.retry_count = 1;
	int_to_scsilun(cmd->device->lun, &lun);
	memcpy(task->ssp_task.LUN, &lun.scsi_lun, 8);
	task->ssp_task.task_attr = sas_scsi_get_task_attr(cmd);
	memcpy(task->ssp_task.cdb, cmd->cmnd, 16);

	task->scatter = scsi_sglist(cmd);
	task->num_scatter = scsi_sg_count(cmd);
	task->total_xfer_len = scsi_bufflen(cmd);
	task->data_dir = cmd->sc_data_direction;

	task->task_done = sas_scsi_task_done;

	return task;
}

static int sas_queue_up(struct sas_task *task)
{
	struct sas_ha_struct *sas_ha = task->dev->port->ha;
	struct scsi_core *core = &sas_ha->core;
	unsigned long flags;
	LIST_HEAD(list);

	spin_lock_irqsave(&core->task_queue_lock, flags);
	if (sas_ha->lldd_queue_size < core->task_queue_size + 1) {
		spin_unlock_irqrestore(&core->task_queue_lock, flags);
		return -SAS_QUEUE_FULL;
	}
	list_add_tail(&task->list, &core->task_queue);
	core->task_queue_size += 1;
	spin_unlock_irqrestore(&core->task_queue_lock, flags);
	wake_up_process(core->queue_thread);

	return 0;
}

static inline int dev_is_sata(struct domain_device *dev)
{
	return (dev->rphy->identify.target_port_protocols & SAS_PROTOCOL_SATA);
}

/**
 * sas_queuecommand -- Enqueue a command for processing
 * @parameters: See SCSI Core documentation
 *
 * Note: XXX: Remove the host unlock/lock pair when SCSI Core can
 * call us without holding an IRQ spinlock...
 */
int sas_queuecommand(struct scsi_cmnd *cmd,
		     void (*scsi_done)(struct scsi_cmnd *))
{
	int res = 0;
	struct domain_device *dev = cmd_to_domain_dev(cmd);
	struct Scsi_Host *host = cmd->device->host;
	struct sas_internal *i = to_sas_internal(host->transportt);

	spin_unlock_irq(host->host_lock);

	{
		struct sas_ha_struct *sas_ha = dev->port->ha;
		struct sas_task *task;

		if (dev_is_sata(dev)) {
			res = ata_sas_queuecmd(cmd, scsi_done,
					       dev->sata_dev.ap);
			goto out;
		}

		res = -ENOMEM;
		task = sas_create_task(cmd, dev, GFP_ATOMIC);
		if (!task)
			goto out;

		cmd->scsi_done = scsi_done;
		/* Queue up, Direct Mode or Task Collector Mode. */
		if (sas_ha->lldd_max_execute_num < 2)
			res = i->dft->lldd_execute_task(task, 1, GFP_ATOMIC);
		else
			res = sas_queue_up(task);

		/* Examine */
		if (res) {
			SAS_DPRINTK("lldd_execute_task returned: %d\n", res);
			ASSIGN_SAS_TASK(cmd, NULL);
			sas_free_task(task);
			if (res == -SAS_QUEUE_FULL) {
				cmd->result = DID_SOFT_ERROR << 16; /* retry */
				res = 0;
				scsi_done(cmd);
			}
			goto out;
		}
	}
out:
	spin_lock_irq(host->host_lock);
	return res;
}

static void sas_scsi_clear_queue_lu(struct list_head *error_q, struct scsi_cmnd *my_cmd)
{
	struct scsi_cmnd *cmd, *n;

	list_for_each_entry_safe(cmd, n, error_q, eh_entry) {
		if (cmd == my_cmd)
			list_del_init(&cmd->eh_entry);
	}
}

static void sas_scsi_clear_queue_I_T(struct list_head *error_q,
				     struct domain_device *dev)
{
	struct scsi_cmnd *cmd, *n;

	list_for_each_entry_safe(cmd, n, error_q, eh_entry) {
		struct domain_device *x = cmd_to_domain_dev(cmd);

		if (x == dev)
			list_del_init(&cmd->eh_entry);
	}
}

static void sas_scsi_clear_queue_port(struct list_head *error_q,
				      struct asd_sas_port *port)
{
	struct scsi_cmnd *cmd, *n;

	list_for_each_entry_safe(cmd, n, error_q, eh_entry) {
		struct domain_device *dev = cmd_to_domain_dev(cmd);
		struct asd_sas_port *x = dev->port;

		if (x == port)
			list_del_init(&cmd->eh_entry);
	}
}

enum task_disposition {
	TASK_IS_DONE,
	TASK_IS_ABORTED,
	TASK_IS_AT_LU,
	TASK_IS_NOT_AT_LU,
	TASK_ABORT_FAILED,
};

static enum task_disposition sas_scsi_find_task(struct sas_task *task)
{
	struct sas_ha_struct *ha = task->dev->port->ha;
	unsigned long flags;
	int i, res;
	struct sas_internal *si =
		to_sas_internal(task->dev->port->ha->core.shost->transportt);

	if (ha->lldd_max_execute_num > 1) {
		struct scsi_core *core = &ha->core;
		struct sas_task *t, *n;

		spin_lock_irqsave(&core->task_queue_lock, flags);
		list_for_each_entry_safe(t, n, &core->task_queue, list) {
			if (task == t) {
				list_del_init(&t->list);
				spin_unlock_irqrestore(&core->task_queue_lock,
						       flags);
				SAS_DPRINTK("%s: task 0x%p aborted from "
					    "task_queue\n",
					    __FUNCTION__, task);
				return TASK_IS_ABORTED;
			}
		}
		spin_unlock_irqrestore(&core->task_queue_lock, flags);
	}

	for (i = 0; i < 5; i++) {
		SAS_DPRINTK("%s: aborting task 0x%p\n", __FUNCTION__, task);
		res = si->dft->lldd_abort_task(task);

		spin_lock_irqsave(&task->task_state_lock, flags);
		if (task->task_state_flags & SAS_TASK_STATE_DONE) {
			spin_unlock_irqrestore(&task->task_state_lock, flags);
			SAS_DPRINTK("%s: task 0x%p is done\n", __FUNCTION__,
				    task);
			return TASK_IS_DONE;
		}
		spin_unlock_irqrestore(&task->task_state_lock, flags);

		if (res == TMF_RESP_FUNC_COMPLETE) {
			SAS_DPRINTK("%s: task 0x%p is aborted\n",
				    __FUNCTION__, task);
			return TASK_IS_ABORTED;
		} else if (si->dft->lldd_query_task) {
			SAS_DPRINTK("%s: querying task 0x%p\n",
				    __FUNCTION__, task);
			res = si->dft->lldd_query_task(task);
			switch (res) {
			case TMF_RESP_FUNC_SUCC:
				SAS_DPRINTK("%s: task 0x%p at LU\n",
					    __FUNCTION__, task);
				return TASK_IS_AT_LU;
			case TMF_RESP_FUNC_COMPLETE:
				SAS_DPRINTK("%s: task 0x%p not at LU\n",
					    __FUNCTION__, task);
				return TASK_IS_NOT_AT_LU;
			case TMF_RESP_FUNC_FAILED:
                                SAS_DPRINTK("%s: task 0x%p failed to abort\n",
                                                __FUNCTION__, task);
                                return TASK_ABORT_FAILED;
                        }

		}
	}
	return res;
}

static int sas_recover_lu(struct domain_device *dev, struct scsi_cmnd *cmd)
{
	int res = TMF_RESP_FUNC_FAILED;
	struct scsi_lun lun;
	struct sas_internal *i =
		to_sas_internal(dev->port->ha->core.shost->transportt);

	int_to_scsilun(cmd->device->lun, &lun);

	SAS_DPRINTK("eh: device %llx LUN %x has the task\n",
		    SAS_ADDR(dev->sas_addr),
		    cmd->device->lun);

	if (i->dft->lldd_abort_task_set)
		res = i->dft->lldd_abort_task_set(dev, lun.scsi_lun);

	if (res == TMF_RESP_FUNC_FAILED) {
		if (i->dft->lldd_clear_task_set)
			res = i->dft->lldd_clear_task_set(dev, lun.scsi_lun);
	}

	if (res == TMF_RESP_FUNC_FAILED) {
		if (i->dft->lldd_lu_reset)
			res = i->dft->lldd_lu_reset(dev, lun.scsi_lun);
	}

	return res;
}

static int sas_recover_I_T(struct domain_device *dev)
{
	int res = TMF_RESP_FUNC_FAILED;
	struct sas_internal *i =
		to_sas_internal(dev->port->ha->core.shost->transportt);

	SAS_DPRINTK("I_T nexus reset for dev %016llx\n",
		    SAS_ADDR(dev->sas_addr));

	if (i->dft->lldd_I_T_nexus_reset)
		res = i->dft->lldd_I_T_nexus_reset(dev);

	return res;
}

/* Find the sas_phy that's attached to this device */
struct sas_phy *find_local_sas_phy(struct domain_device *dev)
{
	struct domain_device *pdev = dev->parent;
	struct ex_phy *exphy = NULL;
	int i;

	/* Directly attached device */
	if (!pdev)
		return dev->port->phy;

	/* Otherwise look in the expander */
	for (i = 0; i < pdev->ex_dev.num_phys; i++)
		if (!memcmp(dev->sas_addr,
			    pdev->ex_dev.ex_phy[i].attached_sas_addr,
			    SAS_ADDR_SIZE)) {
			exphy = &pdev->ex_dev.ex_phy[i];
			break;
		}

	BUG_ON(!exphy);
	return exphy->phy;
}

/* Attempt to send a LUN reset message to a device */
int sas_eh_device_reset_handler(struct scsi_cmnd *cmd)
{
	struct domain_device *dev = cmd_to_domain_dev(cmd);
	struct sas_internal *i =
		to_sas_internal(dev->port->ha->core.shost->transportt);
	struct scsi_lun lun;
	int res;

	int_to_scsilun(cmd->device->lun, &lun);

	if (!i->dft->lldd_lu_reset)
		return FAILED;

	res = i->dft->lldd_lu_reset(dev, lun.scsi_lun);
	if (res == TMF_RESP_FUNC_SUCC || res == TMF_RESP_FUNC_COMPLETE)
		return SUCCESS;

	return FAILED;
}

/* Attempt to send a phy (bus) reset */
int sas_eh_bus_reset_handler(struct scsi_cmnd *cmd)
{
	struct domain_device *dev = cmd_to_domain_dev(cmd);
	struct sas_phy *phy = find_local_sas_phy(dev);
	int res;

	res = sas_phy_reset(phy, 1);
	if (res)
		SAS_DPRINTK("Bus reset of %s failed 0x%x\n",
			    phy->dev.kobj.k_name,
			    res);
	if (res == TMF_RESP_FUNC_SUCC || res == TMF_RESP_FUNC_COMPLETE)
		return SUCCESS;

	return FAILED;
}

/* Try to reset a device */
static int try_to_reset_cmd_device(struct Scsi_Host *shost,
				   struct scsi_cmnd *cmd)
{
	int res;

	if (!shost->hostt->eh_device_reset_handler)
		goto try_bus_reset;

	res = shost->hostt->eh_device_reset_handler(cmd);
	if (res == SUCCESS)
		return res;

try_bus_reset:
	if (shost->hostt->eh_bus_reset_handler)
		return shost->hostt->eh_bus_reset_handler(cmd);

	return FAILED;
}

static int sas_eh_handle_sas_errors(struct Scsi_Host *shost,
				    struct list_head *work_q,
				    struct list_head *done_q)
{
	struct scsi_cmnd *cmd, *n;
	enum task_disposition res = TASK_IS_DONE;
	int tmf_resp, need_reset;
	struct sas_internal *i = to_sas_internal(shost->transportt);
	unsigned long flags;
	struct sas_ha_struct *ha = SHOST_TO_SAS_HA(shost);

Again:
	list_for_each_entry_safe(cmd, n, work_q, eh_entry) {
		struct sas_task *task = TO_SAS_TASK(cmd);

		if (!task)
			continue;

		list_del_init(&cmd->eh_entry);

		spin_lock_irqsave(&task->task_state_lock, flags);
		need_reset = task->task_state_flags & SAS_TASK_NEED_DEV_RESET;
		spin_unlock_irqrestore(&task->task_state_lock, flags);

		SAS_DPRINTK("trying to find task 0x%p\n", task);
		res = sas_scsi_find_task(task);

		cmd->eh_eflags = 0;

		switch (res) {
		case TASK_IS_DONE:
			SAS_DPRINTK("%s: task 0x%p is done\n", __FUNCTION__,
				    task);
			task->task_done(task);
			if (need_reset)
				try_to_reset_cmd_device(shost, cmd);
			continue;
		case TASK_IS_ABORTED:
			SAS_DPRINTK("%s: task 0x%p is aborted\n",
				    __FUNCTION__, task);
			task->task_done(task);
			if (need_reset)
				try_to_reset_cmd_device(shost, cmd);
			continue;
		case TASK_IS_AT_LU:
			SAS_DPRINTK("task 0x%p is at LU: lu recover\n", task);
			tmf_resp = sas_recover_lu(task->dev, cmd);
			if (tmf_resp == TMF_RESP_FUNC_COMPLETE) {
				SAS_DPRINTK("dev %016llx LU %x is "
					    "recovered\n",
					    SAS_ADDR(task->dev),
					    cmd->device->lun);
				task->task_done(task);
				if (need_reset)
					try_to_reset_cmd_device(shost, cmd);
				sas_scsi_clear_queue_lu(work_q, cmd);
				goto Again;
			}
			/* fallthrough */
		case TASK_IS_NOT_AT_LU:
		case TASK_ABORT_FAILED:
			SAS_DPRINTK("task 0x%p is not at LU: I_T recover\n",
				    task);
			tmf_resp = sas_recover_I_T(task->dev);
			if (tmf_resp == TMF_RESP_FUNC_COMPLETE) {
				SAS_DPRINTK("I_T %016llx recovered\n",
					    SAS_ADDR(task->dev->sas_addr));
				task->task_done(task);
				if (need_reset)
					try_to_reset_cmd_device(shost, cmd);
				sas_scsi_clear_queue_I_T(work_q, task->dev);
				goto Again;
			}
			/* Hammer time :-) */
			if (i->dft->lldd_clear_nexus_port) {
				struct asd_sas_port *port = task->dev->port;
				SAS_DPRINTK("clearing nexus for port:%d\n",
					    port->id);
				res = i->dft->lldd_clear_nexus_port(port);
				if (res == TMF_RESP_FUNC_COMPLETE) {
					SAS_DPRINTK("clear nexus port:%d "
						    "succeeded\n", port->id);
					task->task_done(task);
					if (need_reset)
						try_to_reset_cmd_device(shost, cmd);
					sas_scsi_clear_queue_port(work_q,
								  port);
					goto Again;
				}
			}
			if (i->dft->lldd_clear_nexus_ha) {
				SAS_DPRINTK("clear nexus ha\n");
				res = i->dft->lldd_clear_nexus_ha(ha);
				if (res == TMF_RESP_FUNC_COMPLETE) {
					SAS_DPRINTK("clear nexus ha "
						    "succeeded\n");
					task->task_done(task);
					if (need_reset)
						try_to_reset_cmd_device(shost, cmd);
					goto out;
				}
			}
			/* If we are here -- this means that no amount
			 * of effort could recover from errors.  Quite
			 * possibly the HA just disappeared.
			 */
			SAS_DPRINTK("error from  device %llx, LUN %x "
				    "couldn't be recovered in any way\n",
				    SAS_ADDR(task->dev->sas_addr),
				    cmd->device->lun);

			task->task_done(task);
			if (need_reset)
				try_to_reset_cmd_device(shost, cmd);
			goto clear_q;
		}
	}
out:
	return list_empty(work_q);
clear_q:
	SAS_DPRINTK("--- Exit %s -- clear_q\n", __FUNCTION__);
	list_for_each_entry_safe(cmd, n, work_q, eh_entry) {
		struct sas_task *task = TO_SAS_TASK(cmd);
		list_del_init(&cmd->eh_entry);
		task->task_done(task);
	}
	return list_empty(work_q);
}

void sas_scsi_recover_host(struct Scsi_Host *shost)
{
	struct sas_ha_struct *ha = SHOST_TO_SAS_HA(shost);
	unsigned long flags;
	LIST_HEAD(eh_work_q);

	spin_lock_irqsave(shost->host_lock, flags);
	list_splice_init(&shost->eh_cmd_q, &eh_work_q);
	spin_unlock_irqrestore(shost->host_lock, flags);

	SAS_DPRINTK("Enter %s\n", __FUNCTION__);
	/*
	 * Deal with commands that still have SAS tasks (i.e. they didn't
	 * complete via the normal sas_task completion mechanism)
	 */
	if (sas_eh_handle_sas_errors(shost, &eh_work_q, &ha->eh_done_q))
		goto out;

	/*
	 * Now deal with SCSI commands that completed ok but have a an error
	 * code (and hopefully sense data) attached.  This is roughly what
	 * scsi_unjam_host does, but we skip scsi_eh_abort_cmds because any
	 * command we see here has no sas_task and is thus unknown to the HA.
	 */
	if (!scsi_eh_get_sense(&eh_work_q, &ha->eh_done_q))
		scsi_eh_ready_devs(shost, &eh_work_q, &ha->eh_done_q);

out:
	scsi_eh_flush_done_q(&ha->eh_done_q);
	SAS_DPRINTK("--- Exit %s\n", __FUNCTION__);
	return;
}

enum scsi_eh_timer_return sas_scsi_timed_out(struct scsi_cmnd *cmd)
{
	struct sas_task *task = TO_SAS_TASK(cmd);
	unsigned long flags;

	if (!task) {
		cmd->timeout_per_command /= 2;
		SAS_DPRINTK("command 0x%p, task 0x%p, gone: %s\n",
			    cmd, task, (cmd->timeout_per_command ?
			    "EH_RESET_TIMER" : "EH_NOT_HANDLED"));
		if (!cmd->timeout_per_command)
			return EH_NOT_HANDLED;
		return EH_RESET_TIMER;
	}

	spin_lock_irqsave(&task->task_state_lock, flags);
	BUG_ON(task->task_state_flags & SAS_TASK_STATE_ABORTED);
	if (task->task_state_flags & SAS_TASK_STATE_DONE) {
		spin_unlock_irqrestore(&task->task_state_lock, flags);
		SAS_DPRINTK("command 0x%p, task 0x%p, timed out: EH_HANDLED\n",
			    cmd, task);
		return EH_HANDLED;
	}
	if (!(task->task_state_flags & SAS_TASK_AT_INITIATOR)) {
		spin_unlock_irqrestore(&task->task_state_lock, flags);
		SAS_DPRINTK("command 0x%p, task 0x%p, not at initiator: "
			    "EH_RESET_TIMER\n",
			    cmd, task);
		return EH_RESET_TIMER;
	}
	task->task_state_flags |= SAS_TASK_STATE_ABORTED;
	spin_unlock_irqrestore(&task->task_state_lock, flags);

	SAS_DPRINTK("command 0x%p, task 0x%p, timed out: EH_NOT_HANDLED\n",
		    cmd, task);

	return EH_NOT_HANDLED;
}


static enum ata_completion_errors sas_to_ata_err(struct task_status_struct *ts)
{
	/* Cheesy attempt to translate SAS errors into ATA.  Hah! */

	/* transport error */
	if (ts->resp == SAS_TASK_UNDELIVERED)
		return AC_ERR_ATA_BUS;

	/* ts->resp == SAS_TASK_COMPLETE */
	/* task delivered, what happened afterwards? */
	switch (ts->stat) {
		case SAS_DEV_NO_RESPONSE:
			return AC_ERR_TIMEOUT;

		case SAS_INTERRUPTED:
		case SAS_PHY_DOWN:
		case SAS_NAK_R_ERR:
			return AC_ERR_ATA_BUS;


		case SAS_DATA_UNDERRUN:
			/*
			 * Some programs that use the taskfile interface
			 * (smartctl in particular) can cause underrun
			 * problems.  Ignore these errors, perhaps at our
			 * peril.
			 */
			return 0;

		case SAS_DATA_OVERRUN:
		case SAS_QUEUE_FULL:
		case SAS_DEVICE_UNKNOWN:
		case SAS_SG_ERR:
			return AC_ERR_INVALID;

		case SAM_CHECK_COND:
		case SAS_OPEN_TO:
		case SAS_OPEN_REJECT:
		case SAS_PROTO_RESPONSE:
			SAS_DPRINTK("%s: Saw error %d.  What to do?\n",
				    __FUNCTION__, ts->stat);
			return AC_ERR_OTHER;

		case SAS_ABORTED_TASK:
			return AC_ERR_DEV;

		default:
			return 0;
	}
}

static void sas_ata_task_done(struct sas_task *task)
{
	struct ata_queued_cmd *qc = task->uldd_task;
	struct domain_device *dev = qc->ap->private_data;
	struct task_status_struct *stat = &task->task_status;
	struct ata_task_resp *resp = (struct ata_task_resp *)stat->buf;
	enum ata_completion_errors ac;

	ac = sas_to_ata_err(stat);
	if (ac) {
		SAS_DPRINTK("%s: SAS error %x\n", __FUNCTION__, stat->stat);
		/* We saw a SAS error. Send a vague error. */
		qc->err_mask = ac;
		dev->sata_dev.tf.feature = 0x04; /* status err */
		dev->sata_dev.tf.command = ATA_ERR;
		goto end;
	}

	ata_tf_from_fis(resp->ending_fis, &dev->sata_dev.tf);
	qc->err_mask |= ac_err_mask(dev->sata_dev.tf.command);
	dev->sata_dev.sstatus = resp->sstatus;
	dev->sata_dev.serror = resp->serror;
	dev->sata_dev.scontrol = resp->scontrol;
	dev->sata_dev.ap->sactive = resp->sactive;
end:
	ata_qc_complete(qc);
	list_del_init(&task->list);
	sas_free_task(task);
}

int sas_ioctl(struct scsi_device *sdev, int cmd, void __user *arg)
{
	struct domain_device *dev = sdev_to_domain_dev(sdev);

	if (dev_is_sata(dev))
		return ata_scsi_ioctl(sdev, cmd, arg);

	return -EINVAL;
}

static unsigned int sas_ata_qc_issue(struct ata_queued_cmd *qc)
{
	int res = -ENOMEM;
	struct sas_task *task;
	struct domain_device *dev = qc->ap->private_data;
	struct sas_ha_struct *sas_ha = dev->port->ha;
	struct Scsi_Host *host = sas_ha->core.shost;
	struct sas_internal *i = to_sas_internal(host->transportt);
	struct scatterlist *sg;
	unsigned int num = 0;
	unsigned int xfer = 0;

	task = sas_alloc_task(GFP_ATOMIC);
	if (!task)
		goto out;
	task->dev = dev;
	task->task_proto = SAS_PROTOCOL_STP;
	task->task_done = sas_ata_task_done;

	ata_tf_to_fis(&qc->tf, (u8*)&task->ata_task.fis, 0);
	task->uldd_task = qc;
	if (is_atapi_taskfile(&qc->tf)) {
		memcpy(task->ata_task.atapi_packet, qc->cdb, ATAPI_CDB_LEN);
		task->total_xfer_len = qc->nbytes + qc->pad_len;
		task->num_scatter = qc->pad_len ? qc->n_elem + 1 : qc->n_elem;
	} else {
		ata_for_each_sg(sg, qc) {
			num++;
			xfer += sg->length;
		}

		task->total_xfer_len = xfer;
		task->num_scatter = num;
	}

	task->data_dir = qc->dma_dir;
	task->scatter = qc->__sg;
	task->ata_task.retry_count = 1;
	task->task_state_flags = SAS_TASK_STATE_PENDING;

	if (qc->tf.protocol == ATA_PROT_DMA)
		task->ata_task.dma_xfer = 1;

	if (sas_ha->lldd_max_execute_num < 2)
		res = i->dft->lldd_execute_task(task, 1, GFP_ATOMIC);
	else
		res = sas_queue_up(task);

	/* Examine */
	if (res) {
		SAS_DPRINTK("lldd_execute_task returned: %d\n", res);

		sas_free_task(task);
		if (res == -SAS_QUEUE_FULL)
			return -ENOMEM;
	}

out:
	return res;
}

static u8 sas_ata_check_status(struct ata_port *ap)
{
	struct domain_device *dev = ap->private_data;
	return dev->sata_dev.tf.command;
}

static void sas_ata_phy_reset(struct ata_port *ap)
{
	struct domain_device *dev = ap->private_data;
	struct sas_internal *i =
		to_sas_internal(dev->port->ha->core.shost->transportt);
	int res = 0;

	if (i->dft->lldd_I_T_nexus_reset)
		res = i->dft->lldd_I_T_nexus_reset(dev);

	if (res)
		SAS_DPRINTK("%s: Unable to reset I T nexus?\n", __FUNCTION__);

	switch (dev->sata_dev.command_set) {
		case ATA_COMMAND_SET:
			SAS_DPRINTK("%s: Found ATA device.\n", __FUNCTION__);
			ap->device[0].class = ATA_DEV_ATA;
			break;
		case ATAPI_COMMAND_SET:
			SAS_DPRINTK("%s: Found ATAPI device.\n", __FUNCTION__);
			ap->device[0].class = ATA_DEV_ATAPI;
			break;
		default:
			SAS_DPRINTK("%s: Unknown SATA command set: %d.\n",
				    __FUNCTION__,
				    dev->sata_dev.command_set);
			ap->device[0].class = ATA_DEV_ATA;
			break;
	}

	ap->cbl = ATA_CBL_SATA;
}

static void sas_ata_post_internal(struct ata_queued_cmd *qc)
{
	if (qc->flags & ATA_QCFLAG_FAILED)
		qc->err_mask |= AC_ERR_OTHER;

	if (qc->err_mask)
		SAS_DPRINTK("%s: Failure; reset phy!\n", __FUNCTION__);
}

static void sas_ata_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
{
	struct domain_device *dev = ap->private_data;
	memcpy(tf, &dev->sata_dev.tf, sizeof (*tf));
}

static void sas_ata_scr_write(struct ata_port *ap, unsigned int sc_reg_in,
			      u32 val)
{
	struct domain_device *dev = ap->private_data;

	SAS_DPRINTK("STUB %s\n", __FUNCTION__);
	switch (sc_reg_in) {
		case SCR_STATUS:
			dev->sata_dev.sstatus = val;
			break;
		case SCR_CONTROL:
			dev->sata_dev.scontrol = val;
			break;
		case SCR_ERROR:
			dev->sata_dev.serror = val;
			break;
		case SCR_ACTIVE:
			dev->sata_dev.ap->sactive = val;
			break;
	}
}

static u32 sas_ata_scr_read(struct ata_port *ap, unsigned int sc_reg_in)
{
	struct domain_device *dev = ap->private_data;

	SAS_DPRINTK("STUB %s\n", __FUNCTION__);
	switch (sc_reg_in) {
		case SCR_STATUS:
			return dev->sata_dev.sstatus;
		case SCR_CONTROL:
			return dev->sata_dev.scontrol;
		case SCR_ERROR:
			return dev->sata_dev.serror;
		case SCR_ACTIVE:
			return dev->sata_dev.ap->sactive;
		default:
			return 0xffffffffU;
	}
}

static struct ata_port_operations sas_sata_ops = {
	.port_disable		= ata_port_disable,
	.check_status		= sas_ata_check_status,
	.check_altstatus	= sas_ata_check_status,
	.dev_select		= ata_noop_dev_select,
	.phy_reset		= sas_ata_phy_reset,
	.post_internal_cmd	= sas_ata_post_internal,
	.tf_read		= sas_ata_tf_read,
	.qc_prep		= ata_noop_qc_prep,
	.qc_issue		= sas_ata_qc_issue,
	.port_start		= ata_sas_port_start,
	.port_stop		= ata_sas_port_stop,
	.scr_read		= sas_ata_scr_read,
	.scr_write		= sas_ata_scr_write
};

static struct ata_port_info sata_port_info = {
	.flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY | ATA_FLAG_SATA_RESET |
		ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA,
	.pio_mask = 0x1f, /* PIO0-4 */
	.mwdma_mask = 0x07, /* MWDMA0-2 */
	.udma_mask = ATA_UDMA6,
	.port_ops = &sas_sata_ops
};

struct domain_device *sas_find_dev_by_rphy(struct sas_rphy *rphy)
{
	struct Scsi_Host *shost = dev_to_shost(rphy->dev.parent);
	struct sas_ha_struct *ha = SHOST_TO_SAS_HA(shost);
	struct domain_device *found_dev = NULL;
	int i;
	unsigned long flags;

	spin_lock_irqsave(&ha->phy_port_lock, flags);
	for (i = 0; i < ha->num_phys; i++) {
		struct asd_sas_port *port = ha->sas_port[i];
		struct domain_device *dev;

		spin_lock(&port->dev_list_lock);
		list_for_each_entry(dev, &port->dev_list, dev_list_node) {
			if (rphy == dev->rphy) {
				found_dev = dev;
				spin_unlock(&port->dev_list_lock);
				goto found;
			}
		}
		spin_unlock(&port->dev_list_lock);
	}
 found:
	spin_unlock_irqrestore(&ha->phy_port_lock, flags);

	return found_dev;
}

static inline struct domain_device *sas_find_target(struct scsi_target *starget)
{
	struct sas_rphy *rphy = dev_to_rphy(starget->dev.parent);

	return sas_find_dev_by_rphy(rphy);
}

int sas_target_alloc(struct scsi_target *starget)
{
	struct Scsi_Host *shost = dev_to_shost(&starget->dev);
	struct sas_ha_struct *ha = SHOST_TO_SAS_HA(shost);
	struct domain_device *found_dev = sas_find_target(starget);

	if (!found_dev)
		return -ENODEV;

	if (dev_is_sata(found_dev)) {
		struct ata_port *ap;

		ata_host_init(&found_dev->sata_dev.ata_host,
			      &ha->pcidev->dev,
			      sata_port_info.flags,
			      &sas_sata_ops);
		ap = ata_sas_port_alloc(&found_dev->sata_dev.ata_host,
					&sata_port_info,
					shost);
		if (!ap) {
			SAS_DPRINTK("ata_sas_port_alloc failed.\n");
			return -ENODEV;
		}

		ap->private_data = found_dev;
		ap->cbl = ATA_CBL_SATA;
		found_dev->sata_dev.ap = ap;
	}

	starget->hostdata = found_dev;
	return 0;
}

#define SAS_DEF_QD 32
#define SAS_MAX_QD 64

int sas_slave_configure(struct scsi_device *scsi_dev)
{
	struct domain_device *dev = sdev_to_domain_dev(scsi_dev);
	struct sas_ha_struct *sas_ha;

	BUG_ON(dev->rphy->identify.device_type != SAS_END_DEVICE);

	if (dev_is_sata(dev)) {
		ata_sas_slave_configure(scsi_dev, dev->sata_dev.ap);
		return 0;
	}

	sas_ha = dev->port->ha;

	sas_read_port_mode_page(scsi_dev);

	if (scsi_dev->tagged_supported) {
		scsi_set_tag_type(scsi_dev, MSG_SIMPLE_TAG);
		scsi_activate_tcq(scsi_dev, SAS_DEF_QD);
	} else {
		SAS_DPRINTK("device %llx, LUN %x doesn't support "
			    "TCQ\n", SAS_ADDR(dev->sas_addr),
			    scsi_dev->lun);
		scsi_dev->tagged_supported = 0;
		scsi_set_tag_type(scsi_dev, 0);
		scsi_deactivate_tcq(scsi_dev, 1);
	}

	scsi_dev->allow_restart = 1;

	return 0;
}

void sas_slave_destroy(struct scsi_device *scsi_dev)
{
	struct domain_device *dev = sdev_to_domain_dev(scsi_dev);

	if (dev_is_sata(dev))
		ata_port_disable(dev->sata_dev.ap);
}

int sas_change_queue_depth(struct scsi_device *scsi_dev, int new_depth)
{
	int res = min(new_depth, SAS_MAX_QD);

	if (scsi_dev->tagged_supported)
		scsi_adjust_queue_depth(scsi_dev, scsi_get_tag_type(scsi_dev),
					res);
	else {
		struct domain_device *dev = sdev_to_domain_dev(scsi_dev);
		sas_printk("device %llx LUN %x queue depth changed to 1\n",
			   SAS_ADDR(dev->sas_addr),
			   scsi_dev->lun);
		scsi_adjust_queue_depth(scsi_dev, 0, 1);
		res = 1;
	}

	return res;
}

int sas_change_queue_type(struct scsi_device *scsi_dev, int qt)
{
	if (!scsi_dev->tagged_supported)
		return 0;

	scsi_deactivate_tcq(scsi_dev, 1);

	scsi_set_tag_type(scsi_dev, qt);
	scsi_activate_tcq(scsi_dev, scsi_dev->queue_depth);

	return qt;
}

int sas_bios_param(struct scsi_device *scsi_dev,
			  struct block_device *bdev,
			  sector_t capacity, int *hsc)
{
	hsc[0] = 255;
	hsc[1] = 63;
	sector_div(capacity, 255*63);
	hsc[2] = capacity;

	return 0;
}

/* ---------- Task Collector Thread implementation ---------- */

static void sas_queue(struct sas_ha_struct *sas_ha)
{
	struct scsi_core *core = &sas_ha->core;
	unsigned long flags;
	LIST_HEAD(q);
	int can_queue;
	int res;
	struct sas_internal *i = to_sas_internal(core->shost->transportt);

	spin_lock_irqsave(&core->task_queue_lock, flags);
	while (!kthread_should_stop() &&
	       !list_empty(&core->task_queue)) {

		can_queue = sas_ha->lldd_queue_size - core->task_queue_size;
		if (can_queue >= 0) {
			can_queue = core->task_queue_size;
			list_splice_init(&core->task_queue, &q);
		} else {
			struct list_head *a, *n;

			can_queue = sas_ha->lldd_queue_size;
			list_for_each_safe(a, n, &core->task_queue) {
				list_move_tail(a, &q);
				if (--can_queue == 0)
					break;
			}
			can_queue = sas_ha->lldd_queue_size;
		}
		core->task_queue_size -= can_queue;
		spin_unlock_irqrestore(&core->task_queue_lock, flags);
		{
			struct sas_task *task = list_entry(q.next,
							   struct sas_task,
							   list);
			list_del_init(&q);
			res = i->dft->lldd_execute_task(task, can_queue,
							GFP_KERNEL);
			if (unlikely(res))
				__list_add(&q, task->list.prev, &task->list);
		}
		spin_lock_irqsave(&core->task_queue_lock, flags);
		if (res) {
			list_splice_init(&q, &core->task_queue); /*at head*/
			core->task_queue_size += can_queue;
		}
	}
	spin_unlock_irqrestore(&core->task_queue_lock, flags);
}

/**
 * sas_queue_thread -- The Task Collector thread
 * @_sas_ha: pointer to struct sas_ha
 */
static int sas_queue_thread(void *_sas_ha)
{
	struct sas_ha_struct *sas_ha = _sas_ha;

	while (1) {
		set_current_state(TASK_INTERRUPTIBLE);
		schedule();
		sas_queue(sas_ha);
		if (kthread_should_stop())
			break;
	}

	return 0;
}

int sas_init_queue(struct sas_ha_struct *sas_ha)
{
	struct scsi_core *core = &sas_ha->core;

	spin_lock_init(&core->task_queue_lock);
	core->task_queue_size = 0;
	INIT_LIST_HEAD(&core->task_queue);

	core->queue_thread = kthread_run(sas_queue_thread, sas_ha,
					 "sas_queue_%d", core->shost->host_no);
	if (IS_ERR(core->queue_thread))
		return PTR_ERR(core->queue_thread);
	return 0;
}

void sas_shutdown_queue(struct sas_ha_struct *sas_ha)
{
	unsigned long flags;
	struct scsi_core *core = &sas_ha->core;
	struct sas_task *task, *n;

	kthread_stop(core->queue_thread);

	if (!list_empty(&core->task_queue))
		SAS_DPRINTK("HA: %llx: scsi core task queue is NOT empty!?\n",
			    SAS_ADDR(sas_ha->sas_addr));

	spin_lock_irqsave(&core->task_queue_lock, flags);
	list_for_each_entry_safe(task, n, &core->task_queue, list) {
		struct scsi_cmnd *cmd = task->uldd_task;

		list_del_init(&task->list);

		ASSIGN_SAS_TASK(cmd, NULL);
		sas_free_task(task);
		cmd->result = DID_ABORT << 16;
		cmd->scsi_done(cmd);
	}
	spin_unlock_irqrestore(&core->task_queue_lock, flags);
}

/*
 * Call the LLDD task abort routine directly.  This function is intended for
 * use by upper layers that need to tell the LLDD to abort a task.
 */
int __sas_task_abort(struct sas_task *task)
{
	struct sas_internal *si =
		to_sas_internal(task->dev->port->ha->core.shost->transportt);
	unsigned long flags;
	int res;

	spin_lock_irqsave(&task->task_state_lock, flags);
	if (task->task_state_flags & SAS_TASK_STATE_ABORTED ||
	    task->task_state_flags & SAS_TASK_STATE_DONE) {
		spin_unlock_irqrestore(&task->task_state_lock, flags);
		SAS_DPRINTK("%s: Task %p already finished.\n", __FUNCTION__,
			    task);
		return 0;
	}
	task->task_state_flags |= SAS_TASK_STATE_ABORTED;
	spin_unlock_irqrestore(&task->task_state_lock, flags);

	if (!si->dft->lldd_abort_task)
		return -ENODEV;

	res = si->dft->lldd_abort_task(task);

	spin_lock_irqsave(&task->task_state_lock, flags);
	if ((task->task_state_flags & SAS_TASK_STATE_DONE) ||
	    (res == TMF_RESP_FUNC_COMPLETE))
	{
		spin_unlock_irqrestore(&task->task_state_lock, flags);
		task->task_done(task);
		return 0;
	}

	if (!(task->task_state_flags & SAS_TASK_STATE_DONE))
		task->task_state_flags &= ~SAS_TASK_STATE_ABORTED;
	spin_unlock_irqrestore(&task->task_state_lock, flags);

	return -EAGAIN;
}

/*
 * Tell an upper layer that it needs to initiate an abort for a given task.
 * This should only ever be called by an LLDD.
 */
void sas_task_abort(struct sas_task *task)
{
	struct scsi_cmnd *sc = task->uldd_task;

	/* Escape for libsas internal commands */
	if (!sc) {
		if (!del_timer(&task->timer))
			return;
		task->timer.function(task->timer.data);
		return;
	}

	scsi_req_abort_cmd(sc);
	scsi_schedule_eh(sc->device->host);
}

int sas_slave_alloc(struct scsi_device *scsi_dev)
{
	struct domain_device *dev = sdev_to_domain_dev(scsi_dev);

	if (dev_is_sata(dev))
		return ata_sas_port_init(dev->sata_dev.ap);

	return 0;
}

void sas_target_destroy(struct scsi_target *starget)
{
	struct domain_device *found_dev = sas_find_target(starget);

	if (!found_dev)
		return;

	if (dev_is_sata(found_dev))
		ata_sas_port_destroy(found_dev->sata_dev.ap);

	return;
}

EXPORT_SYMBOL_GPL(sas_queuecommand);
EXPORT_SYMBOL_GPL(sas_target_alloc);
EXPORT_SYMBOL_GPL(sas_slave_configure);
EXPORT_SYMBOL_GPL(sas_slave_destroy);
EXPORT_SYMBOL_GPL(sas_change_queue_depth);
EXPORT_SYMBOL_GPL(sas_change_queue_type);
EXPORT_SYMBOL_GPL(sas_bios_param);
EXPORT_SYMBOL_GPL(__sas_task_abort);
EXPORT_SYMBOL_GPL(sas_task_abort);
EXPORT_SYMBOL_GPL(sas_phy_reset);
EXPORT_SYMBOL_GPL(sas_phy_enable);
EXPORT_SYMBOL_GPL(sas_eh_device_reset_handler);
EXPORT_SYMBOL_GPL(sas_eh_bus_reset_handler);
EXPORT_SYMBOL_GPL(sas_slave_alloc);
EXPORT_SYMBOL_GPL(sas_target_destroy);
EXPORT_SYMBOL_GPL(sas_ioctl);