src/sys/dev/pci/mmuagp.c

/*-
 * Copyright (c) 2004, 2005 Jung-uk Kim <jkim@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/device.h>
#include <sys/rwlock.h>

#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/agpvar.h>
#include <dev/pci/agpreg.h>

#include <dev/pci/pcidevs.h>

#include <machine/bus.h>

/*
 * AMD64 GART registers
 */
#define	AGP_AMD64_APCTRL		0x90
#define	AGP_AMD64_APBASE		0x94
#define	AGP_AMD64_ATTBASE		0x98
#define	AGP_AMD64_CACHECTRL		0x9c
#define	AGP_AMD64_APCTRL_GARTEN		0x00000001
#define	AGP_AMD64_APCTRL_SIZE_MASK	0x0000000e
#define	AGP_AMD64_APCTRL_DISGARTCPU	0x00000010
#define	AGP_AMD64_APCTRL_DISGARTIO	0x00000020
#define	AGP_AMD64_APCTRL_DISWLKPRB	0x00000040
#define	AGP_AMD64_APBASE_MASK		0x00007fff
#define	AGP_AMD64_ATTBASE_MASK		0xfffffff0
#define	AGP_AMD64_CACHECTRL_INVGART	0x00000001
#define	AGP_AMD64_CACHECTRL_PTEERR	0x00000002

/*
 * NVIDIA nForce3 registers
 */
#define AGP_AMD64_NVIDIA_0_APBASE	0x10
#define AGP_AMD64_NVIDIA_1_APBASE1	0x50
#define AGP_AMD64_NVIDIA_1_APLIMIT1	0x54
#define AGP_AMD64_NVIDIA_1_APSIZE	0xa8
#define AGP_AMD64_NVIDIA_1_APBASE2	0xd8
#define AGP_AMD64_NVIDIA_1_APLIMIT2	0xdc

/*
 * ULi M1689 registers
 */
#define AGP_AMD64_ULI_APBASE		0x10
#define AGP_AMD64_ULI_HTT_FEATURE	0x50
#define AGP_AMD64_ULI_ENU_SCR		0x54


#define	AMD64_MAX_MCTRL		8

/* XXX nForce3 requires secondary AGP bridge at 0:11:0. */
#define AGP_AMD64_NVIDIA_PCITAG(pc)	pci_make_tag(pc, 0, 11, 0)
/* XXX Some VIA bridge requires secondary AGP bridge at 0:1:0. */
#define AGP_AMD64_VIA_PCITAG(pc)	pci_make_tag(pc, 0, 1, 0)


int	mmuagp_probe(struct device *, void *, void *);
void	mmuagp_attach(struct device *, struct device *, void *);
bus_size_t mmuagp_get_aperture(void *);
int	mmuagp_set_aperture(void *, bus_size_t);
void	mmuagp_bind_page(void *, bus_addr_t, paddr_t, int);
void	mmuagp_unbind_page(void *, bus_addr_t);
void	mmuagp_flush_tlb(void *);

void	mmuagp_apbase_fixup(void *);

void	mmuagp_uli_init(void *);
int	mmuagp_uli_set_aperture(void *, bus_size_t);

int	mmuagp_nvidia_match(const struct pci_attach_args *, uint16_t);
void	mmuagp_nvidia_init(void *);
int	mmuagp_nvidia_set_aperture(void *, bus_size_t);

int	mmuagp_via_match(const struct pci_attach_args *);
void	mmuagp_via_init(void *);
int	mmuagp_via_set_aperture(void *, bus_size_t);

struct mmuagp_softc {
	struct device		 dev;
	struct agp_softc	*agpdev;
	struct agp_gatt		*gatt;
	bus_addr_t		 msc_apaddr;
	bus_size_t		 msc_apsize;
	uint32_t		 apbase;
	pcitag_t		 ctrl_tag;	/* use NVIDIA and VIA */
	pcitag_t		 mctrl_tag[AMD64_MAX_MCTRL];
	pci_chipset_tag_t	 msc_pc;
	pcitag_t		 msc_tag;
	int			 n_mctrl;
};

const struct cfattach mmuagp_ca = {
        sizeof(struct mmuagp_softc), mmuagp_probe, mmuagp_attach
};

struct cfdriver mmuagp_cd = {
	NULL, "mmuagp", DV_DULL
};

const struct agp_methods mmuagp_methods = {
	mmuagp_bind_page,
	mmuagp_unbind_page,
	mmuagp_flush_tlb,
};

int
mmuagp_probe(struct device *parent, void *match, void *aux)
{
	struct agp_attach_args	*aa = aux;
	struct pci_attach_args	*pa = aa->aa_pa;

	/* Must be a pchb, don't attach to iommu-style agp devs */
	if (agpbus_probe(aa) == 0)
		return (0);

	switch (PCI_VENDOR(pa->pa_id)) {
	case PCI_VENDOR_ALI:
		switch (PCI_PRODUCT(pa->pa_id)) {
		case PCI_PRODUCT_ALI_M1689:
			return (1);
		}
		break;
	case PCI_VENDOR_AMD:
		switch (PCI_PRODUCT(pa->pa_id)) {
		case PCI_PRODUCT_AMD_8151_SC:
			return (1);
		}
		break;
	case PCI_VENDOR_NVIDIA:
		switch (PCI_PRODUCT(pa->pa_id)) {
		case PCI_PRODUCT_NVIDIA_NFORCE3_PCHB:
			return (mmuagp_nvidia_match(pa,
			    PCI_PRODUCT_NVIDIA_NFORCE3_PPB2));
			/* NOTREACHED */
		case PCI_PRODUCT_NVIDIA_NFORCE3_250_PCHB:
			return (mmuagp_nvidia_match(pa,
			    PCI_PRODUCT_NVIDIA_NFORCE3_250_AGP));
			/* NOTREACHED */
		}
		break;
	case PCI_VENDOR_SIS:
		switch (PCI_PRODUCT(pa->pa_id)) {
		case PCI_PRODUCT_SIS_755:
		case PCI_PRODUCT_SIS_760:
			return (1);
		}
		break;
	case PCI_VENDOR_VIATECH:
		switch (PCI_PRODUCT(pa->pa_id)) {
		case PCI_PRODUCT_VIATECH_K8M800_0:
		case PCI_PRODUCT_VIATECH_K8T890_0:
		case PCI_PRODUCT_VIATECH_K8HTB_0:
		case PCI_PRODUCT_VIATECH_K8HTB:
			return (1);
		}
		break;
	}

	return (0);
}

int
mmuagp_nvidia_match(const struct pci_attach_args *pa, uint16_t devid)
{
	pcitag_t	tag;
	pcireg_t	reg;

	tag = AGP_AMD64_NVIDIA_PCITAG(pa->pa_pc);

	reg = pci_conf_read(pa->pa_pc, tag, PCI_CLASS_REG);
	if (PCI_CLASS(reg) != PCI_CLASS_BRIDGE ||
	    PCI_SUBCLASS(reg) != PCI_SUBCLASS_BRIDGE_PCI)
		return 0;

	reg = pci_conf_read(pa->pa_pc, tag, PCI_ID_REG);
	if (PCI_VENDOR(reg) != PCI_VENDOR_NVIDIA || PCI_PRODUCT(reg) != devid)
		return 0;

	return 1;
}

int
mmuagp_via_match(const struct pci_attach_args *pa)
{
	pcitag_t tag;
	pcireg_t reg;

	tag = AGP_AMD64_VIA_PCITAG(pa->pa_pc);

	reg = pci_conf_read(pa->pa_pc, tag, PCI_CLASS_REG);
	if (PCI_CLASS(reg) != PCI_CLASS_BRIDGE ||
	    PCI_SUBCLASS(reg) != PCI_SUBCLASS_BRIDGE_PCI)
		return 0;

	reg = pci_conf_read(pa->pa_pc, tag, PCI_ID_REG);
	if (PCI_VENDOR(reg) != PCI_VENDOR_VIATECH ||
	    PCI_PRODUCT(reg) != PCI_PRODUCT_VIATECH_K8HTB_AGP)
		return 0;

	return 1;
}

void
mmuagp_attach(struct device *parent, struct device *self, void *aux)
{
	struct mmuagp_softc	*msc = (struct mmuagp_softc *)self ;
	struct agp_attach_args	*aa = aux;
	struct pci_attach_args	*pa = aa->aa_pa;
	struct agp_gatt		*gatt;
	int			 (*set_aperture)(void *, bus_size_t) = NULL;
	pcireg_t		 id, attbase, apctrl;
	pcitag_t		 tag;
	int			 maxdevs, i, n;

	if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, AGP_APBASE,
	    PCI_MAPREG_TYPE_MEM, &msc->msc_apaddr, NULL, NULL) != 0) {
		printf(": can't get aperture info\n");
		return;
	}

	msc->msc_pc = pa->pa_pc;
	msc->msc_tag = pa->pa_tag;

	maxdevs = pci_bus_maxdevs(pa->pa_pc, 0);
	for (i = 0, n = 0; i < maxdevs && n < AMD64_MAX_MCTRL; i++) {
		tag = pci_make_tag(pa->pa_pc, 0, i, 3);
		id = pci_conf_read(pa->pa_pc, tag, PCI_ID_REG);
		if (PCI_VENDOR(id) == PCI_VENDOR_AMD &&
		    PCI_PRODUCT(id) ==  PCI_PRODUCT_AMD_0F_MISC) {
			msc->mctrl_tag[n] = tag;
			n++;
		}
	}
	if (n == 0) {
		printf(": no Miscellaneous Control units found\n");
		return;
	}
	msc->n_mctrl = n;

	printf(": %d Miscellaneous Control unit(s) found", msc->n_mctrl);

	msc->msc_apsize = mmuagp_get_aperture(msc);

	for (;;) {
		gatt = agp_alloc_gatt(pa->pa_dmat, msc->msc_apsize);
		if (gatt != NULL)
			break;

		/*
		 * Probably failed to alloc contiguous memory. Try reducing the
		 * aperture so that the gatt size reduces.
		 */
		msc->msc_apsize /= 2;
		if (mmuagp_set_aperture(msc, msc->msc_apsize)) {
			printf(" can't set aperture size\n");
			return;
		}
	}
	msc->gatt = gatt;

	switch (PCI_VENDOR(pa->pa_id)) {
	case PCI_VENDOR_ALI:
		mmuagp_uli_init(msc);
		set_aperture = mmuagp_uli_set_aperture;
		break;

	case PCI_VENDOR_NVIDIA:
		msc->ctrl_tag = AGP_AMD64_NVIDIA_PCITAG(pa->pa_pc);
		mmuagp_nvidia_init(msc);
		set_aperture = mmuagp_nvidia_set_aperture;
		break;

	case PCI_VENDOR_VIATECH:
		/* do we have to set the extra bridge too? */
		if (mmuagp_via_match(pa)) {
			msc->ctrl_tag = AGP_AMD64_VIA_PCITAG(pa->pa_pc);
			mmuagp_via_init(msc);
			set_aperture = mmuagp_via_set_aperture;
		}
		break;
	}

	if (set_aperture != NULL) {
		if ((*set_aperture)(msc, msc->msc_apsize)) {
			printf(", failed aperture set\n");
			return;
		}
	}

	/* Install the gatt and enable aperture. */
	attbase = (uint32_t)(gatt->ag_physical >> 8) & AGP_AMD64_ATTBASE_MASK;
	for (i = 0; i < msc->n_mctrl; i++) {
		pci_conf_write(pa->pa_pc, msc->mctrl_tag[i], AGP_AMD64_ATTBASE,
		    attbase);
		apctrl = pci_conf_read(pa->pa_pc, msc->mctrl_tag[i],
		    AGP_AMD64_APCTRL);
		apctrl |= AGP_AMD64_APCTRL_GARTEN;
		apctrl &=
		    ~(AGP_AMD64_APCTRL_DISGARTCPU | AGP_AMD64_APCTRL_DISGARTIO);
		pci_conf_write(pa->pa_pc, msc->mctrl_tag[i], AGP_AMD64_APCTRL,
		    apctrl);
	}

	agp_flush_cache();

	msc->agpdev = (struct agp_softc *)agp_attach_bus(pa, &mmuagp_methods,
	    msc->msc_apaddr, msc->msc_apsize, &msc->dev);
	return;
}


static bus_size_t mmuagp_table[] = {
	0x02000000,	/*   32 MB */
	0x04000000,	/*   64 MB */
	0x08000000,	/*  128 MB */
	0x10000000,	/*  256 MB */
	0x20000000,	/*  512 MB */
	0x40000000,	/* 1024 MB */
	0x80000000,	/* 2048 MB */
};

#define AGP_AMD64_TABLE_SIZE \
	(sizeof(mmuagp_table) / sizeof(mmuagp_table[0]))

bus_size_t
mmuagp_get_aperture(void *sc)
{
	struct mmuagp_softc *msc = sc;
	uint32_t i;

	i = (pci_conf_read(msc->msc_pc, msc->mctrl_tag[0], AGP_AMD64_APCTRL) &
		AGP_AMD64_APCTRL_SIZE_MASK) >> 1;

	if (i >= AGP_AMD64_TABLE_SIZE)
		return 0;

	return mmuagp_table[i];
}

int
mmuagp_set_aperture(void *sc, bus_size_t aperture)
{
	struct mmuagp_softc	*msc = sc;
	uint32_t		 i;
	pcireg_t		 apctrl;
	int			 j;

	for (i = 0; i < AGP_AMD64_TABLE_SIZE; i++)
		if (mmuagp_table[i] == aperture)
			break;
	if (i >= AGP_AMD64_TABLE_SIZE)
		return (EINVAL);

	for (j = 0; j < msc->n_mctrl; j++) {
		apctrl = pci_conf_read(msc->msc_pc, msc->mctrl_tag[0],
		    AGP_AMD64_APCTRL);
		pci_conf_write(msc->msc_pc, msc->mctrl_tag[0], AGP_AMD64_APCTRL,
		    (apctrl & ~(AGP_AMD64_APCTRL_SIZE_MASK)) | (i << 1));
	}

	return (0);
}

void
mmuagp_bind_page(void *sc, bus_addr_t offset, paddr_t physical, int flags)
{
	struct mmuagp_softc	*msc = sc;

	msc->gatt->ag_virtual[(offset - msc->msc_apaddr) >> AGP_PAGE_SHIFT] =
	    (physical & 0xfffff000) | ((physical >> 28) & 0x00000ff0) | 3;
}

void
mmuagp_unbind_page(void *sc, bus_addr_t offset)
{
	struct mmuagp_softc *msc = sc;

	msc->gatt->ag_virtual[(offset - msc->msc_apaddr) >> AGP_PAGE_SHIFT] = 0;
}

void
mmuagp_flush_tlb(void *sc)
{
	struct mmuagp_softc	*msc = sc;
	pcireg_t		 cachectrl;
	int			 i;

	for (i = 0; i < msc->n_mctrl; i++) {
		cachectrl = pci_conf_read(msc->msc_pc, msc->mctrl_tag[i],
		    AGP_AMD64_CACHECTRL);
		pci_conf_write(msc->msc_pc, msc->mctrl_tag[i],
		    AGP_AMD64_CACHECTRL,
		    cachectrl | AGP_AMD64_CACHECTRL_INVGART);
	}
}

void
mmuagp_apbase_fixup(void *sc)
{
	struct mmuagp_softc	*msc = sc;
	uint32_t		 apbase;
	int			 i;

	apbase = pci_conf_read(msc->msc_pc, msc->msc_tag, AGP_APBASE);
	msc->apbase = PCI_MAPREG_MEM_ADDR(apbase);
	apbase = (msc->apbase >> 25) & AGP_AMD64_APBASE_MASK;
	for (i = 0; i < msc->n_mctrl; i++)
		pci_conf_write(msc->msc_pc, msc->mctrl_tag[i], AGP_AMD64_APBASE,
		    apbase);
}

void
mmuagp_uli_init(void *sc)
{
	struct mmuagp_softc *msc = sc;
	pcireg_t apbase;

	mmuagp_apbase_fixup(msc);
	apbase = pci_conf_read(msc->msc_pc, msc->msc_tag,
	    AGP_AMD64_ULI_APBASE);
	pci_conf_write(msc->msc_pc, msc->msc_tag, AGP_AMD64_ULI_APBASE,
	    (apbase & 0x0000000f) | msc->apbase);
	pci_conf_write(msc->msc_pc, msc->msc_tag, AGP_AMD64_ULI_HTT_FEATURE,
	    msc->apbase);
}

int
mmuagp_uli_set_aperture(void *sc, bus_size_t aperture)
{
	struct mmuagp_softc	*msc = sc;

	switch (aperture) {
	case 0x02000000:	/*  32 MB */
	case 0x04000000:	/*  64 MB */
	case 0x08000000:	/* 128 MB */
	case 0x10000000:	/* 256 MB */
		break;
	default:
		return EINVAL;
	}

	pci_conf_write(msc->msc_pc, msc->msc_tag, AGP_AMD64_ULI_ENU_SCR,
	    msc->apbase + aperture - 1);

	return 0;
}

void
mmuagp_nvidia_init(void *sc)
{
	struct mmuagp_softc	*msc = sc;
	pcireg_t		 apbase;

	mmuagp_apbase_fixup(msc);
	apbase = pci_conf_read(msc->msc_pc, msc->msc_tag,
	    AGP_AMD64_NVIDIA_0_APBASE);
	pci_conf_write(msc->msc_pc, msc->msc_tag, AGP_AMD64_NVIDIA_0_APBASE,
	    (apbase & 0x0000000f) | msc->apbase);
	pci_conf_write(msc->msc_pc, msc->ctrl_tag, AGP_AMD64_NVIDIA_1_APBASE1,
	    msc->apbase);
	pci_conf_write(msc->msc_pc, msc->ctrl_tag, AGP_AMD64_NVIDIA_1_APBASE2,
	    msc->apbase);
}

int
mmuagp_nvidia_set_aperture(void *sc, bus_size_t aperture)
{
	struct mmuagp_softc	*msc = sc;
	bus_size_t		 apsize;

	switch (aperture) {
	case 0x02000000:	/*  32 MB */
		apsize = 0x0f;
		break;
	case 0x04000000:	/*  64 MB */
		apsize = 0x0e;
		break;
	case 0x08000000:	/* 128 MB */
		apsize = 0x0c;
		break;
	case 0x10000000:	/* 256 MB */
		apsize = 0x08;
		break;
	case 0x20000000:	/* 512 MB */
		apsize = 0x00;
		break;
	default:
		return (EINVAL);
	}

	pci_conf_write(msc->msc_pc, msc->ctrl_tag, AGP_AMD64_NVIDIA_1_APSIZE,
	    (pci_conf_read(msc->msc_pc, msc->ctrl_tag,
	    AGP_AMD64_NVIDIA_1_APSIZE) & 0xfffffff0) | apsize);
	pci_conf_write(msc->msc_pc, msc->ctrl_tag, AGP_AMD64_NVIDIA_1_APLIMIT1,
	    msc->apbase + aperture - 1);
	pci_conf_write(msc->msc_pc, msc->ctrl_tag, AGP_AMD64_NVIDIA_1_APLIMIT2,
	    msc->apbase + aperture - 1);

	return (0);
}

void
mmuagp_via_init(void *sc)
{
	struct mmuagp_softc	*msc = sc;

	mmuagp_apbase_fixup(sc);
	pci_conf_write(msc->msc_pc, msc->ctrl_tag, AGP3_VIA_ATTBASE,
	    msc->gatt->ag_physical);
	pci_conf_write(msc->msc_pc, msc->ctrl_tag, AGP3_VIA_GARTCTRL,
	    pci_conf_read(msc->msc_pc, msc->ctrl_tag, AGP3_VIA_ATTBASE) |
	        0x180);
}

int
mmuagp_via_set_aperture(void *sc, bus_size_t aperture)
{
	struct mmuagp_softc	*msc = sc;
	bus_size_t		 apsize;

	apsize = ((aperture - 1) >> 20) ^ 0xff;
	if ((((apsize ^ 0xff) << 20) | ((1 << 20) - 1)) + 1 != aperture)
		return (EINVAL);
	pci_conf_write(msc->msc_pc, msc->ctrl_tag, AGP3_VIA_APSIZE,
	    (pci_conf_read(msc->msc_pc, msc->ctrl_tag, AGP3_VIA_APSIZE) &
	        ~0xff) | apsize);

	return 0;
}