src/sys/dev/softraid_raid5.c

/* $OpenBSD: softraid_raid5.c,v 1.32 2021/05/16 15:12:37 deraadt Exp $ */
/*
 * Copyright (c) 2014 Joel Sing <jsing@openbsd.org>
 * Copyright (c) 2009 Marco Peereboom <marco@peereboom.us>
 * Copyright (c) 2009 Jordan Hargrave <jordan@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include "bio.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/buf.h>
#include <sys/device.h>
#include <sys/ioctl.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/disk.h>
#include <sys/rwlock.h>
#include <sys/queue.h>
#include <sys/fcntl.h>
#include <sys/mount.h>
#include <sys/sensors.h>
#include <sys/stat.h>
#include <sys/task.h>
#include <sys/pool.h>
#include <sys/conf.h>
#include <sys/uio.h>

#include <scsi/scsi_all.h>
#include <scsi/scsiconf.h>
#include <scsi/scsi_disk.h>

#include <dev/softraidvar.h>

/* RAID 5 functions. */
int	sr_raid5_create(struct sr_discipline *, struct bioc_createraid *,
	    int, int64_t);
int	sr_raid5_assemble(struct sr_discipline *, struct bioc_createraid *,
	    int, void *);
int	sr_raid5_init(struct sr_discipline *);
int	sr_raid5_rw(struct sr_workunit *);
int	sr_raid5_openings(struct sr_discipline *);
void	sr_raid5_intr(struct buf *);
int	sr_raid5_wu_done(struct sr_workunit *);
void	sr_raid5_set_chunk_state(struct sr_discipline *, int, int);
void	sr_raid5_set_vol_state(struct sr_discipline *);

int	sr_raid5_addio(struct sr_workunit *wu, int, daddr_t, long,
	    void *, int, int, void *);
int	sr_raid5_regenerate(struct sr_workunit *, int, daddr_t, long,
	    void *);
int	sr_raid5_write(struct sr_workunit *, struct sr_workunit *, int, int,
	    daddr_t, long, void *, int, int);
void	sr_raid5_xor(void *, void *, int);

void	sr_raid5_rebuild(struct sr_discipline *);
void	sr_raid5_scrub(struct sr_discipline *);

/* discipline initialisation. */
void
sr_raid5_discipline_init(struct sr_discipline *sd)
{
	/* Fill out discipline members. */
	sd->sd_type = SR_MD_RAID5;
	strlcpy(sd->sd_name, "RAID 5", sizeof(sd->sd_name));
	sd->sd_capabilities = SR_CAP_SYSTEM_DISK | SR_CAP_AUTO_ASSEMBLE |
	    SR_CAP_REBUILD | SR_CAP_REDUNDANT;
	sd->sd_max_wu = SR_RAID5_NOWU + 2;	/* Two for scrub/rebuild. */

	/* Setup discipline specific function pointers. */
	sd->sd_assemble = sr_raid5_assemble;
	sd->sd_create = sr_raid5_create;
	sd->sd_openings = sr_raid5_openings;
	sd->sd_rebuild = sr_raid5_rebuild;
	sd->sd_scsi_rw = sr_raid5_rw;
	sd->sd_scsi_intr = sr_raid5_intr;
	sd->sd_scsi_wu_done = sr_raid5_wu_done;
	sd->sd_set_chunk_state = sr_raid5_set_chunk_state;
	sd->sd_set_vol_state = sr_raid5_set_vol_state;
}

int
sr_raid5_create(struct sr_discipline *sd, struct bioc_createraid *bc,
    int no_chunk, int64_t coerced_size)
{
	if (no_chunk < 3) {
		sr_error(sd->sd_sc, "%s requires three or more chunks",
		    sd->sd_name);
		return EINVAL;
	}

	/*
	 * XXX add variable strip size later even though MAXPHYS is really
	 * the clever value, users like to tinker with that type of stuff.
	 */
	sd->sd_meta->ssdi.ssd_strip_size = MAXPHYS;
	sd->sd_meta->ssdi.ssd_size = (coerced_size &
	    ~(((u_int64_t)sd->sd_meta->ssdi.ssd_strip_size >>
	    DEV_BSHIFT) - 1)) * (no_chunk - 1);

	return sr_raid5_init(sd);
}

int
sr_raid5_assemble(struct sr_discipline *sd, struct bioc_createraid *bc,
    int no_chunk, void *data)
{
	return sr_raid5_init(sd);
}

int
sr_raid5_init(struct sr_discipline *sd)
{
	/* Initialise runtime values. */
	sd->mds.mdd_raid5.sr5_strip_bits =
	    sr_validate_stripsize(sd->sd_meta->ssdi.ssd_strip_size);
	if (sd->mds.mdd_raid5.sr5_strip_bits == -1) {
		sr_error(sd->sd_sc, "invalid strip size");
		return EINVAL;
	}

	sd->sd_max_ccb_per_wu = sd->sd_meta->ssdi.ssd_chunk_no;

	return 0;
}

int
sr_raid5_openings(struct sr_discipline *sd)
{
	/* Two work units per I/O, two for rebuild/scrub. */
	return ((sd->sd_max_wu - 2) >> 1);
}

void
sr_raid5_set_chunk_state(struct sr_discipline *sd, int c, int new_state)
{
	int			old_state, s;

	DNPRINTF(SR_D_STATE, "%s: %s: %s: sr_raid_set_chunk_state %d -> %d\n",
	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname,
	    sd->sd_vol.sv_chunks[c]->src_meta.scmi.scm_devname, c, new_state);

	/* ok to go to splbio since this only happens in error path */
	s = splbio();
	old_state = sd->sd_vol.sv_chunks[c]->src_meta.scm_status;

	/* multiple IOs to the same chunk that fail will come through here */
	if (old_state == new_state)
		goto done;

	switch (old_state) {
	case BIOC_SDONLINE:
		switch (new_state) {
		case BIOC_SDOFFLINE:
		case BIOC_SDSCRUB:
			break;
		default:
			goto die;
		}
		break;

	case BIOC_SDOFFLINE:
		if (new_state == BIOC_SDREBUILD) {
			;
		} else
			goto die;
		break;

	case BIOC_SDSCRUB:
		switch (new_state) {
		case BIOC_SDONLINE:
		case BIOC_SDOFFLINE:
			break;
		default:
			goto die;
		}
		break;

	case BIOC_SDREBUILD:
		switch (new_state) {
		case BIOC_SDONLINE:
		case BIOC_SDOFFLINE:
			break;
		default:
			goto die;
		}
		break;

	default:
die:
		splx(s); /* XXX */
		panic("%s: %s: %s: invalid chunk state transition %d -> %d",
		    DEVNAME(sd->sd_sc),
		    sd->sd_meta->ssd_devname,
		    sd->sd_vol.sv_chunks[c]->src_meta.scmi.scm_devname,
		    old_state, new_state);
		/* NOTREACHED */
	}

	sd->sd_vol.sv_chunks[c]->src_meta.scm_status = new_state;
	sd->sd_set_vol_state(sd);

	sd->sd_must_flush = 1;
	task_add(systq, &sd->sd_meta_save_task);
done:
	splx(s);
}

void
sr_raid5_set_vol_state(struct sr_discipline *sd)
{
	int			states[SR_MAX_STATES];
	int			new_state, i, s, nd;
	int			old_state = sd->sd_vol_status;

	DNPRINTF(SR_D_STATE, "%s: %s: sr_raid_set_vol_state\n",
	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname);

	nd = sd->sd_meta->ssdi.ssd_chunk_no;

	for (i = 0; i < SR_MAX_STATES; i++)
		states[i] = 0;

	for (i = 0; i < nd; i++) {
		s = sd->sd_vol.sv_chunks[i]->src_meta.scm_status;
		if (s >= SR_MAX_STATES)
			panic("%s: %s: %s: invalid chunk state",
			    DEVNAME(sd->sd_sc),
			    sd->sd_meta->ssd_devname,
			    sd->sd_vol.sv_chunks[i]->src_meta.scmi.scm_devname);
		states[s]++;
	}

	if (states[BIOC_SDONLINE] == nd)
		new_state = BIOC_SVONLINE;
	else if (states[BIOC_SDONLINE] < nd - 1)
		new_state = BIOC_SVOFFLINE;
	else if (states[BIOC_SDSCRUB] != 0)
		new_state = BIOC_SVSCRUB;
	else if (states[BIOC_SDREBUILD] != 0)
		new_state = BIOC_SVREBUILD;
	else if (states[BIOC_SDONLINE] == nd - 1)
		new_state = BIOC_SVDEGRADED;
	else {
#ifdef SR_DEBUG
		DNPRINTF(SR_D_STATE, "%s: invalid volume state, old state "
		    "was %d\n", DEVNAME(sd->sd_sc), old_state);
		for (i = 0; i < nd; i++)
			DNPRINTF(SR_D_STATE, "%s: chunk %d status = %d\n",
			    DEVNAME(sd->sd_sc), i,
			    sd->sd_vol.sv_chunks[i]->src_meta.scm_status);
#endif
		panic("invalid volume state");
	}

	DNPRINTF(SR_D_STATE, "%s: %s: sr_raid5_set_vol_state %d -> %d\n",
	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname,
	    old_state, new_state);

	switch (old_state) {
	case BIOC_SVONLINE:
		switch (new_state) {
		case BIOC_SVONLINE: /* can go to same state */
		case BIOC_SVOFFLINE:
		case BIOC_SVDEGRADED:
		case BIOC_SVREBUILD: /* happens on boot */
			break;
		default:
			goto die;
		}
		break;

	case BIOC_SVOFFLINE:
		/* XXX this might be a little too much */
		goto die;

	case BIOC_SVDEGRADED:
		switch (new_state) {
		case BIOC_SVOFFLINE:
		case BIOC_SVREBUILD:
		case BIOC_SVDEGRADED: /* can go to the same state */
			break;
		default:
			goto die;
		}
		break;

	case BIOC_SVBUILDING:
		switch (new_state) {
		case BIOC_SVONLINE:
		case BIOC_SVOFFLINE:
		case BIOC_SVBUILDING: /* can go to the same state */
			break;
		default:
			goto die;
		}
		break;

	case BIOC_SVSCRUB:
		switch (new_state) {
		case BIOC_SVONLINE:
		case BIOC_SVOFFLINE:
		case BIOC_SVDEGRADED:
		case BIOC_SVSCRUB: /* can go to same state */
			break;
		default:
			goto die;
		}
		break;

	case BIOC_SVREBUILD:
		switch (new_state) {
		case BIOC_SVONLINE:
		case BIOC_SVOFFLINE:
		case BIOC_SVDEGRADED:
		case BIOC_SVREBUILD: /* can go to the same state */
			break;
		default:
			goto die;
		}
		break;

	default:
die:
		panic("%s: %s: invalid volume state transition %d -> %d",
		    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname,
		    old_state, new_state);
		/* NOTREACHED */
	}

	sd->sd_vol_status = new_state;
}

static inline int
sr_raid5_chunk_online(struct sr_discipline *sd, int chunk)
{
	switch (sd->sd_vol.sv_chunks[chunk]->src_meta.scm_status) {
	case BIOC_SDONLINE:
	case BIOC_SDSCRUB:
		return 1;
	default:
		return 0;
	}
}

static inline int
sr_raid5_chunk_rebuild(struct sr_discipline *sd, int chunk)
{
	switch (sd->sd_vol.sv_chunks[chunk]->src_meta.scm_status) {
	case BIOC_SDREBUILD:
		return 1;
	default:
		return 0;
	}
}

int
sr_raid5_rw(struct sr_workunit *wu)
{
	struct sr_workunit	*wu_r = NULL;
	struct sr_discipline	*sd = wu->swu_dis;
	struct scsi_xfer	*xs = wu->swu_xs;
	struct sr_chunk		*scp;
	daddr_t			blkno, lba;
	int64_t			chunk_offs, lbaoffs, offset, strip_offs;
	int64_t			strip_bits, strip_no, strip_size;
	int64_t			chunk, no_chunk;
	int64_t			parity, row_size;
	long			length, datalen;
	void			*data;
	int			s;

	/* blkno and scsi error will be handled by sr_validate_io */
	if (sr_validate_io(wu, &blkno, "sr_raid5_rw"))
		goto bad;

	DNPRINTF(SR_D_DIS, "%s: %s sr_raid5_rw %s: blkno %lld size %d\n",
	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname,
	    (xs->flags & SCSI_DATA_IN) ? "read" : "write",
	    (long long)blkno, xs->datalen);

	strip_size = sd->sd_meta->ssdi.ssd_strip_size;
	strip_bits = sd->mds.mdd_raid5.sr5_strip_bits;
	no_chunk = sd->sd_meta->ssdi.ssd_chunk_no - 1;
	row_size = (no_chunk << strip_bits) >> DEV_BSHIFT;

	data = xs->data;
	datalen = xs->datalen;
	lbaoffs	= blkno << DEV_BSHIFT;

	if (xs->flags & SCSI_DATA_OUT) {
		if ((wu_r = sr_scsi_wu_get(sd, SCSI_NOSLEEP)) == NULL){
			printf("%s: %s failed to get read work unit",
			    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname);
			goto bad;
		}
		wu_r->swu_state = SR_WU_INPROGRESS;
		wu_r->swu_flags |= SR_WUF_DISCIPLINE;
	}

	wu->swu_blk_start = 0;
	while (datalen != 0) {
		strip_no = lbaoffs >> strip_bits;
		strip_offs = lbaoffs & (strip_size - 1);
		chunk_offs = (strip_no / no_chunk) << strip_bits;
		offset = chunk_offs + strip_offs;

		/* get size remaining in this stripe */
		length = MIN(strip_size - strip_offs, datalen);

		/*
		 * Map disk offset to data and parity chunks, using a left
		 * asymmetric algorithm for the parity assignment.
		 */
		chunk = strip_no % no_chunk;
		parity = no_chunk - ((strip_no / no_chunk) % (no_chunk + 1));
		if (chunk >= parity)
			chunk++;

		lba = offset >> DEV_BSHIFT;

		/* XXX big hammer.. exclude I/O from entire stripe */
		if (wu->swu_blk_start == 0)
			wu->swu_blk_start = (strip_no / no_chunk) * row_size;
		wu->swu_blk_end = (strip_no / no_chunk) * row_size +
		    (row_size - 1);

		scp = sd->sd_vol.sv_chunks[chunk];
		if (xs->flags & SCSI_DATA_IN) {
			switch (scp->src_meta.scm_status) {
			case BIOC_SDONLINE:
			case BIOC_SDSCRUB:
				/*
				 * Chunk is online, issue a single read
				 * request.
				 */
				if (sr_raid5_addio(wu, chunk, lba, length,
				    data, xs->flags, 0, NULL))
					goto bad;
				break;
			case BIOC_SDOFFLINE:
			case BIOC_SDREBUILD:
			case BIOC_SDHOTSPARE:
				if (sr_raid5_regenerate(wu, chunk, lba,
				    length, data))
					goto bad;
				break;
			default:
				printf("%s: is offline, can't read\n",
				    DEVNAME(sd->sd_sc));
				goto bad;
			}
		} else {
			if (sr_raid5_write(wu, wu_r, chunk, parity, lba,
			    length, data, xs->flags, 0))
				goto bad;
		}

		/* advance to next block */
		lbaoffs += length;
		datalen -= length;
		data += length;
	}

	s = splbio();
	if (wu_r) {
		if (wu_r->swu_io_count > 0) {
			/* collide write request with reads */
			wu_r->swu_blk_start = wu->swu_blk_start;
			wu_r->swu_blk_end = wu->swu_blk_end;

			wu->swu_state = SR_WU_DEFERRED;
			wu_r->swu_collider = wu;
			TAILQ_INSERT_TAIL(&sd->sd_wu_defq, wu, swu_link);

			wu = wu_r;
		} else {
			sr_scsi_wu_put(sd, wu_r);
		}
	}
	splx(s);

	sr_schedule_wu(wu);

	return (0);

bad:
	/* wu is unwound by sr_wu_put */
	if (wu_r)
		sr_scsi_wu_put(sd, wu_r);
	return (1);
}

int
sr_raid5_regenerate(struct sr_workunit *wu, int chunk, daddr_t blkno,
    long len, void *data)
{
	struct sr_discipline	*sd = wu->swu_dis;
	int			i;

	/*
	 * Regenerate a block on a RAID 5 volume by xoring the data and parity
	 * from all of the remaining online chunks. This requires the parity
	 * to already be correct.
	 */

	DNPRINTF(SR_D_DIS, "%s: %s sr_raid5_regenerate chunk %d offline, "
	    "regenerating block %llu\n",
	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname, chunk, blkno);

	memset(data, 0, len);
	for (i = 0; i < sd->sd_meta->ssdi.ssd_chunk_no; i++) {
		if (i == chunk)
			continue;
		if (!sr_raid5_chunk_online(sd, i))
			goto bad;
		if (sr_raid5_addio(wu, i, blkno, len, NULL, SCSI_DATA_IN,
		    0, data))
			goto bad;
	}
	return (0);

bad:
	return (1);
}

int
sr_raid5_write(struct sr_workunit *wu, struct sr_workunit *wu_r, int chunk,
    int parity, daddr_t blkno, long len, void *data, int xsflags,
    int ccbflags)
{
	struct sr_discipline	*sd = wu->swu_dis;
	struct scsi_xfer	*xs = wu->swu_xs;
	void			*xorbuf;
	int			chunk_online, chunk_rebuild;
	int			parity_online, parity_rebuild;
	int			other_offline = 0, other_rebuild = 0;
	int			i;

	/*
	 * Perform a write to a RAID 5 volume. This write routine does not
	 * require the parity to already be correct and will operate on a
	 * uninitialised volume.
	 *
	 * There are four possible cases:
	 *
	 * 1) All data chunks and parity are online. In this case we read the
	 *    data from all data chunks, except the one we are writing to, in
	 *    order to calculate and write the new parity.
	 *
	 * 2) The parity chunk is offline. In this case we only need to write
	 *    to the data chunk. No parity calculation is required.
	 *
	 * 3) The data chunk is offline. In this case we read the data from all
	 *    online chunks in order to calculate and write the new parity.
	 *    This is the same as (1) except we do not write the data chunk.
	 *
	 * 4) A different data chunk is offline. The new parity is calculated
	 *    by taking the existing parity, xoring the original data and
	 *    xoring in the new data. This requires that the parity already be
	 *    correct, which it will be if any of the data chunks has
	 *    previously been written.
	 *
	 * There is an additional complication introduced by a chunk that is
	 * being rebuilt. If this is the data or parity chunk, then we want
	 * to write to it as per normal. If it is another data chunk then we
	 * need to presume that it has not yet been regenerated and use the
	 * same method as detailed in (4) above.
	 */

	DNPRINTF(SR_D_DIS, "%s: %s sr_raid5_write chunk %i parity %i "
	    "blkno %llu\n", DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname,
	    chunk, parity, (unsigned long long)blkno);

	chunk_online = sr_raid5_chunk_online(sd, chunk);
	chunk_rebuild = sr_raid5_chunk_rebuild(sd, chunk);
	parity_online = sr_raid5_chunk_online(sd, parity);
	parity_rebuild = sr_raid5_chunk_rebuild(sd, parity);

	for (i = 0; i < sd->sd_meta->ssdi.ssd_chunk_no; i++) {
		if (i == chunk || i == parity)
			continue;
		if (sr_raid5_chunk_rebuild(sd, i))
			other_rebuild = 1;
		else if (!sr_raid5_chunk_online(sd, i))
			other_offline = 1;
	}

	DNPRINTF(SR_D_DIS, "%s: %s chunk online %d, parity online %d, "
	    "other offline %d\n", DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname,
	    chunk_online, parity_online, other_offline);

	if (!parity_online && !parity_rebuild)
		goto data_write;

	xorbuf = sr_block_get(sd, len);
	if (xorbuf == NULL)
		goto bad;
	memcpy(xorbuf, data, len);

	if (other_offline || other_rebuild) {

		/*
		 * XXX - If we can guarantee that this LBA has been scrubbed
		 * then we can also take this faster path.
		 */

		/* Read in existing data and existing parity. */
		if (sr_raid5_addio(wu_r, chunk, blkno, len, NULL,
		    SCSI_DATA_IN, 0, xorbuf))
			goto bad;
		if (sr_raid5_addio(wu_r, parity, blkno, len, NULL,
		    SCSI_DATA_IN, 0, xorbuf))
			goto bad;

	} else {

		/* Read in existing data from all other chunks. */
		for (i = 0; i < sd->sd_meta->ssdi.ssd_chunk_no; i++) {
			if (i == chunk || i == parity)
				continue;
			if (sr_raid5_addio(wu_r, i, blkno, len, NULL,
			    SCSI_DATA_IN, 0, xorbuf))
				goto bad;
		}

	}

	/* Write new parity. */
	if (sr_raid5_addio(wu, parity, blkno, len, xorbuf, xs->flags,
	    SR_CCBF_FREEBUF, NULL))
		goto bad;

data_write:
	/* Write new data. */
	if (chunk_online || chunk_rebuild)
		if (sr_raid5_addio(wu, chunk, blkno, len, data, xs->flags,
		    0, NULL))
			goto bad;

	return (0);

bad:
	return (1);
}

void
sr_raid5_intr(struct buf *bp)
{
	struct sr_ccb		*ccb = (struct sr_ccb *)bp;
	struct sr_workunit	*wu = ccb->ccb_wu;
	struct sr_discipline	*sd = wu->swu_dis;
	int			s;

	DNPRINTF(SR_D_INTR, "%s: sr_raid5_intr bp %p xs %p\n",
	    DEVNAME(sd->sd_sc), bp, wu->swu_xs);

	s = splbio();
	sr_ccb_done(ccb);

	/* XXX - Should this be done via the taskq? */

	/* XOR data to result. */
	if (ccb->ccb_state == SR_CCB_OK && ccb->ccb_opaque)
		sr_raid5_xor(ccb->ccb_opaque, ccb->ccb_buf.b_data,
		    ccb->ccb_buf.b_bcount);

	/* Free allocated data buffer. */
	if (ccb->ccb_flags & SR_CCBF_FREEBUF) {
		sr_block_put(sd, ccb->ccb_buf.b_data, ccb->ccb_buf.b_bcount);
		ccb->ccb_buf.b_data = NULL;
	}

	sr_wu_done(wu);
	splx(s);
}

int
sr_raid5_wu_done(struct sr_workunit *wu)
{
	struct sr_discipline	*sd = wu->swu_dis;
	struct scsi_xfer	*xs = wu->swu_xs;

	/* XXX - we have no way of propagating errors... */
	if (wu->swu_flags & (SR_WUF_DISCIPLINE | SR_WUF_REBUILD))
		return SR_WU_OK;

	/* XXX - This is insufficient for RAID 5. */
	if (wu->swu_ios_succeeded > 0) {
		xs->error = XS_NOERROR;
		return SR_WU_OK;
	}

	if (xs->flags & SCSI_DATA_IN) {
		printf("%s: retrying read on block %lld\n",
		    sd->sd_meta->ssd_devname, (long long)wu->swu_blk_start);
		sr_wu_release_ccbs(wu);
		wu->swu_state = SR_WU_RESTART;
		if (sd->sd_scsi_rw(wu) == 0)
			return SR_WU_RESTART;
	} else {
		/* XXX - retry write if we just went from online to degraded. */
		printf("%s: permanently fail write on block %lld\n",
		    sd->sd_meta->ssd_devname, (long long)wu->swu_blk_start);
	}

	wu->swu_state = SR_WU_FAILED;
	xs->error = XS_DRIVER_STUFFUP;

	return SR_WU_FAILED;
}

int
sr_raid5_addio(struct sr_workunit *wu, int chunk, daddr_t blkno,
    long len, void *data, int xsflags, int ccbflags, void *xorbuf)
{
	struct sr_discipline	*sd = wu->swu_dis;
	struct sr_ccb		*ccb;

	DNPRINTF(SR_D_DIS, "sr_raid5_addio: %s chunk %d block %lld "
	    "length %ld %s\n", (xsflags & SCSI_DATA_IN) ? "read" : "write",
	    chunk, (long long)blkno, len, xorbuf ? "X0R" : "-");

	/* Allocate temporary buffer. */
	if (data == NULL) {
		data = sr_block_get(sd, len);
		if (data == NULL)
			return (-1);
		ccbflags |= SR_CCBF_FREEBUF;
	}

	ccb = sr_ccb_rw(sd, chunk, blkno, len, data, xsflags, ccbflags);
	if (ccb == NULL) {
		if (ccbflags & SR_CCBF_FREEBUF)
			sr_block_put(sd, data, len);
		return (-1);
	}
	ccb->ccb_opaque = xorbuf;
	sr_wu_enqueue_ccb(wu, ccb);

	return (0);
}

void
sr_raid5_xor(void *a, void *b, int len)
{
	uint32_t		*xa = a, *xb = b;

	len >>= 2;
	while (len--)
		*xa++ ^= *xb++;
}

void
sr_raid5_rebuild(struct sr_discipline *sd)
{
	int64_t strip_no, strip_size, strip_bits, i, restart;
	int64_t chunk_count, chunk_strips, chunk_lba, chunk_size, row_size;
	struct sr_workunit *wu_r, *wu_w;
	int s, slept, percent = 0, old_percent = -1;
	int rebuild_chunk = -1;
	void *xorbuf;

	/* Find the rebuild chunk. */
	for (i = 0; i < sd->sd_meta->ssdi.ssd_chunk_no; i++) {
		if (sr_raid5_chunk_rebuild(sd, i)) {
			rebuild_chunk = i;
			break;
		}
	}
	if (rebuild_chunk == -1)
		goto bad;

	strip_size = sd->sd_meta->ssdi.ssd_strip_size;
	strip_bits = sd->mds.mdd_raid5.sr5_strip_bits;
	chunk_count = sd->sd_meta->ssdi.ssd_chunk_no - 1;
	chunk_size = sd->sd_meta->ssdi.ssd_size / chunk_count;
	chunk_strips = (chunk_size << DEV_BSHIFT) >> strip_bits;
	row_size = (chunk_count << strip_bits) >> DEV_BSHIFT;

	DNPRINTF(SR_D_REBUILD, "%s: %s sr_raid5_rebuild volume size = %lld, "
	    "chunk count = %lld, chunk size = %lld, chunk strips = %lld, "
	    "row size = %lld\n", DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname,
	    sd->sd_meta->ssdi.ssd_size, chunk_count, chunk_size, chunk_strips,
	    row_size);

	restart = sd->sd_meta->ssd_rebuild / row_size;
	if (restart > chunk_strips) {
		printf("%s: bogus rebuild restart offset, starting from 0\n",
		    DEVNAME(sd->sd_sc));
		restart = 0;
	}
	if (restart != 0) {
		percent = sr_rebuild_percent(sd);
		printf("%s: resuming rebuild on %s at %d%%\n",
		    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname, percent);
	}

	for (strip_no = restart; strip_no < chunk_strips; strip_no++) {
		chunk_lba = (strip_size >> DEV_BSHIFT) * strip_no;

		DNPRINTF(SR_D_REBUILD, "%s: %s rebuild strip %lld, "
		    "chunk lba = %lld\n", DEVNAME(sd->sd_sc),
		    sd->sd_meta->ssd_devname, strip_no, chunk_lba);

		wu_w = sr_scsi_wu_get(sd, 0);
		wu_r = sr_scsi_wu_get(sd, 0);

		xorbuf = sr_block_get(sd, strip_size);
		if (xorbuf == NULL)
			goto bad;
		if (sr_raid5_regenerate(wu_r, rebuild_chunk, chunk_lba,
		    strip_size, xorbuf))
			goto bad;
		if (sr_raid5_addio(wu_w, rebuild_chunk, chunk_lba, strip_size,
		    xorbuf, SCSI_DATA_OUT, SR_CCBF_FREEBUF, NULL))
			goto bad;

		/* Collide write work unit with read work unit. */
		wu_r->swu_state = SR_WU_INPROGRESS;
		wu_r->swu_flags |= SR_WUF_REBUILD;
		wu_w->swu_state = SR_WU_DEFERRED;
		wu_w->swu_flags |= SR_WUF_REBUILD | SR_WUF_WAKEUP;
		wu_r->swu_collider = wu_w;

		/* Block I/O to this strip while we rebuild it. */
		wu_r->swu_blk_start = (strip_no / chunk_count) * row_size;
		wu_r->swu_blk_end = wu_r->swu_blk_start + row_size - 1;
		wu_w->swu_blk_start = wu_r->swu_blk_start;
		wu_w->swu_blk_end = wu_r->swu_blk_end;

		DNPRINTF(SR_D_REBUILD, "%s: %s rebuild swu_blk_start = %lld, "
		    "swu_blk_end = %lld\n", DEVNAME(sd->sd_sc),
		    sd->sd_meta->ssd_devname,
		    wu_r->swu_blk_start, wu_r->swu_blk_end);

		s = splbio();
		TAILQ_INSERT_TAIL(&sd->sd_wu_defq, wu_w, swu_link);
		splx(s);

		sr_schedule_wu(wu_r);

		slept = 0;
		while ((wu_w->swu_flags & SR_WUF_REBUILDIOCOMP) == 0) {
			tsleep_nsec(wu_w, PRIBIO, "sr_rebuild", INFSLP);
			slept = 1;
		}
		if (!slept) {
			tsleep_nsec(sd->sd_sc, PWAIT, "sr_yield",
			    MSEC_TO_NSEC(1));
		}

		sr_scsi_wu_put(sd, wu_r);
		sr_scsi_wu_put(sd, wu_w);

		sd->sd_meta->ssd_rebuild = chunk_lba * chunk_count;

		percent = sr_rebuild_percent(sd);
		if (percent != old_percent && strip_no != chunk_strips - 1) {
			if (sr_meta_save(sd, SR_META_DIRTY))
				printf("%s: could not save metadata to %s\n",
				    DEVNAME(sd->sd_sc),
				    sd->sd_meta->ssd_devname);
			old_percent = percent;
		}

		if (sd->sd_reb_abort)
			goto abort;
	}

	DNPRINTF(SR_D_REBUILD, "%s: %s rebuild complete\n", DEVNAME(sd->sd_sc),
	    sd->sd_meta->ssd_devname);

	/* all done */
	sd->sd_meta->ssd_rebuild = 0;
	for (i = 0; i < sd->sd_meta->ssdi.ssd_chunk_no; i++) {
		if (sd->sd_vol.sv_chunks[i]->src_meta.scm_status ==
		    BIOC_SDREBUILD) {
			sd->sd_set_chunk_state(sd, i, BIOC_SDONLINE);
			break;
		}
	}

	return;

abort:
	if (sr_meta_save(sd, SR_META_DIRTY))
		printf("%s: could not save metadata to %s\n",
		    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname);
bad:
	return;
}

#if 0
void
sr_raid5_scrub(struct sr_discipline *sd)
{
	int64_t strip_no, strip_size, no_chunk, parity, max_strip, strip_bits;
	int64_t i;
	struct sr_workunit *wu_r, *wu_w;
	int s, slept;
	void *xorbuf;

	wu_w = sr_scsi_wu_get(sd, 0);
	wu_r = sr_scsi_wu_get(sd, 0);

	no_chunk = sd->sd_meta->ssdi.ssd_chunk_no - 1;
	strip_size = sd->sd_meta->ssdi.ssd_strip_size;
	strip_bits = sd->mds.mdd_raid5.sr5_strip_bits;
	max_strip = sd->sd_meta->ssdi.ssd_size >> strip_bits;

	for (strip_no = 0; strip_no < max_strip; strip_no++) {
		parity = no_chunk - ((strip_no / no_chunk) % (no_chunk + 1));

		xorbuf = sr_block_get(sd, strip_size);
		for (i = 0; i <= no_chunk; i++) {
			if (i != parity)
				sr_raid5_addio(wu_r, i, 0xBADCAFE, strip_size,
				    NULL, SCSI_DATA_IN, 0, xorbuf);
		}
		sr_raid5_addio(wu_w, parity, 0xBADCAFE, strip_size, xorbuf,
		    SCSI_DATA_OUT, SR_CCBF_FREEBUF, NULL);

		wu_r->swu_flags |= SR_WUF_REBUILD;

		/* Collide wu_w with wu_r */
		wu_w->swu_state = SR_WU_DEFERRED;
		wu_w->swu_flags |= SR_WUF_REBUILD | SR_WUF_WAKEUP;
		wu_r->swu_collider = wu_w;

		s = splbio();
		TAILQ_INSERT_TAIL(&sd->sd_wu_defq, wu_w, swu_link);
		splx(s);

		wu_r->swu_state = SR_WU_INPROGRESS;
		sr_schedule_wu(wu_r);

		slept = 0;
		while ((wu_w->swu_flags & SR_WUF_REBUILDIOCOMP) == 0) {
			tsleep_nsec(wu_w, PRIBIO, "sr_scrub", INFSLP);
			slept = 1;
		}
		if (!slept) {
			tsleep_nsec(sd->sd_sc, PWAIT, "sr_yield",
			    MSEC_TO_NSEC(1));
		}
	}
}
#endif