SecBSD/src
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
src/sys/arch/arm64/dev/agintc.c

2084 lines
54 KiB
C
Raw Blame History

/* $OpenBSD: agintc.c,v 1.57 2024/06/19 22:10:45 patrick Exp $ */
/*
* Copyright (c) 2007, 2009, 2011, 2017 Dale Rahn <drahn@dalerahn.com>
* Copyright (c) 2018 Mark Kettenis <kettenis@openbsd.org>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
* This is a device driver for the GICv3/GICv4 IP from ARM as specified
* in IHI0069C, an example of this hardware is the GIC 500.
*/
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/queue.h>
#include <sys/malloc.h>
#include <sys/device.h>
#include <sys/evcount.h>
#include <machine/bus.h>
#include <machine/cpufunc.h>
#include <machine/fdt.h>
#include <dev/ofw/fdt.h>
#include <dev/ofw/openfirm.h>
#include <machine/simplebusvar.h>
#define ICC_PMR s3_0_c4_c6_0
#define ICC_IAR0 s3_0_c12_c8_0
#define ICC_EOIR0 s3_0_c12_c8_1
#define ICC_HPPIR0 s3_0_c12_c8_2
#define ICC_BPR0 s3_0_c12_c8_3
#define ICC_DIR s3_0_c12_c11_1
#define ICC_RPR s3_0_c12_c11_3
#define ICC_SGI1R s3_0_c12_c11_5
#define ICC_SGI0R s3_0_c12_c11_7
#define ICC_IAR1 s3_0_c12_c12_0
#define ICC_EOIR1 s3_0_c12_c12_1
#define ICC_HPPIR1 s3_0_c12_c12_2
#define ICC_BPR1 s3_0_c12_c12_3
#define ICC_CTLR s3_0_c12_c12_4
#define ICC_SRE_EL1 s3_0_c12_c12_5
#define ICC_SRE_EL1_EN 0x7
#define ICC_IGRPEN0 s3_0_c12_c12_6
#define ICC_IGRPEN1 s3_0_c12_c12_7
#define _STR(x) #x
#define STR(x) _STR(x)
/* distributor registers */
#define GICD_CTLR 0x0000
/* non-secure */
#define GICD_CTLR_RWP (1U << 31)
#define GICD_CTLR_EnableGrp1 (1 << 0)
#define GICD_CTLR_EnableGrp1A (1 << 1)
#define GICD_CTLR_ARE_NS (1 << 4)
#define GICD_CTLR_DS (1 << 6)
#define GICD_TYPER 0x0004
#define GICD_TYPER_MBIS (1 << 16)
#define GICD_TYPER_LPIS (1 << 17)
#define GICD_TYPER_ITLINE_M 0x1f
#define GICD_IIDR 0x0008
#define GICD_SETSPI_NSR 0x0040
#define GICD_CLRSPI_NSR 0x0048
#define GICD_IGROUPR(i) (0x0080 + (IRQ_TO_REG32(i) * 4))
#define GICD_ISENABLER(i) (0x0100 + (IRQ_TO_REG32(i) * 4))
#define GICD_ICENABLER(i) (0x0180 + (IRQ_TO_REG32(i) * 4))
#define GICD_ISPENDR(i) (0x0200 + (IRQ_TO_REG32(i) * 4))
#define GICD_ICPENDR(i) (0x0280 + (IRQ_TO_REG32(i) * 4))
#define GICD_ISACTIVER(i) (0x0300 + (IRQ_TO_REG32(i) * 4))
#define GICD_ICACTIVER(i) (0x0380 + (IRQ_TO_REG32(i) * 4))
#define GICD_IPRIORITYR(i) (0x0400 + (i))
#define GICD_ICFGR(i) (0x0c00 + (IRQ_TO_REG16(i) * 4))
#define GICD_ICFGR_TRIG_LEVEL(i) (0x0 << (IRQ_TO_REG16BIT(i) * 2))
#define GICD_ICFGR_TRIG_EDGE(i) (0x2 << (IRQ_TO_REG16BIT(i) * 2))
#define GICD_ICFGR_TRIG_MASK(i) (0x2 << (IRQ_TO_REG16BIT(i) * 2))
#define GICD_IGRPMODR(i) (0x0d00 + (IRQ_TO_REG32(i) * 4))
#define GICD_NSACR(i) (0x0e00 + (IRQ_TO_REG16(i) * 4))
#define GICD_IROUTER(i) (0x6000 + ((i) * 8))
/* redistributor registers */
#define GICR_CTLR 0x00000
#define GICR_CTLR_RWP ((1U << 31) | (1 << 3))
#define GICR_CTLR_ENABLE_LPIS (1 << 0)
#define GICR_IIDR 0x00004
#define GICR_TYPER 0x00008
#define GICR_TYPER_LAST (1 << 4)
#define GICR_TYPER_VLPIS (1 << 1)
#define GICR_WAKER 0x00014
#define GICR_WAKER_X31 (1U << 31)
#define GICR_WAKER_CHILDRENASLEEP (1 << 2)
#define GICR_WAKER_PROCESSORSLEEP (1 << 1)
#define GICR_WAKER_X0 (1 << 0)
#define GICR_PROPBASER 0x00070
#define GICR_PROPBASER_ISH (1ULL << 10)
#define GICR_PROPBASER_IC_NORM_NC (1ULL << 7)
#define GICR_PENDBASER 0x00078
#define GICR_PENDBASER_PTZ (1ULL << 62)
#define GICR_PENDBASER_ISH (1ULL << 10)
#define GICR_PENDBASER_IC_NORM_NC (1ULL << 7)
#define GICR_IGROUPR0 0x10080
#define GICR_ISENABLE0 0x10100
#define GICR_ICENABLE0 0x10180
#define GICR_ISPENDR0 0x10200
#define GICR_ICPENDR0 0x10280
#define GICR_ISACTIVE0 0x10300
#define GICR_ICACTIVE0 0x10380
#define GICR_IPRIORITYR(i) (0x10400 + (i))
#define GICR_ICFGR0 0x10c00
#define GICR_ICFGR1 0x10c04
#define GICR_IGRPMODR0 0x10d00
#define GICR_PROP_SIZE (64 * 1024)
#define GICR_PROP_GROUP1 (1 << 1)
#define GICR_PROP_ENABLE (1 << 0)
#define GICR_PEND_SIZE (64 * 1024)
#define PPI_BASE 16
#define SPI_BASE 32
#define LPI_BASE 8192
#define IRQ_TO_REG32(i) (((i) >> 5) & 0x1f)
#define IRQ_TO_REG32BIT(i) ((i) & 0x1f)
#define IRQ_TO_REG16(i) (((i) >> 4) & 0x3f)
#define IRQ_TO_REG16BIT(i) ((i) & 0xf)
#define IRQ_ENABLE 1
#define IRQ_DISABLE 0
struct agintc_mbi_range {
int mr_base;
int mr_span;
void **mr_mbi;
};
struct agintc_lpi_info {
struct agintc_msi_softc *li_msic;
struct cpu_info *li_ci;
uint32_t li_deviceid;
uint32_t li_eventid;
struct intrhand *li_ih;
};
struct agintc_softc {
struct simplebus_softc sc_sbus;
struct intrq *sc_handler;
struct agintc_lpi_info **sc_lpi;
bus_space_tag_t sc_iot;
bus_space_handle_t sc_d_ioh;
bus_space_handle_t *sc_r_ioh;
bus_space_handle_t sc_redist_base;
bus_dma_tag_t sc_dmat;
uint16_t *sc_processor;
int sc_cpuremap[MAXCPUS];
int sc_nintr;
int sc_nlpi;
bus_addr_t sc_mbi_addr;
int sc_mbi_nranges;
struct agintc_mbi_range *sc_mbi_ranges;
int sc_prio_shift;
int sc_pmr_shift;
int sc_rk3399_quirk;
struct evcount sc_spur;
int sc_ncells;
int sc_num_redist;
struct agintc_dmamem *sc_prop;
struct agintc_dmamem *sc_pend;
struct interrupt_controller sc_ic;
int sc_ipi_num[3]; /* id for each ipi */
int sc_ipi_reason[MAXCPUS]; /* cause of ipi */
void *sc_ipi_irq[3]; /* irqhandle for each ipi */
};
struct agintc_softc *agintc_sc;
struct intrhand {
TAILQ_ENTRY(intrhand) ih_list; /* link on intrq list */
int (*ih_func)(void *); /* handler */
void *ih_arg; /* arg for handler */
int ih_ipl; /* IPL_* */
int ih_flags;
int ih_irq; /* IRQ number */
struct evcount ih_count;
char *ih_name;
struct cpu_info *ih_ci; /* CPU the IRQ runs on */
};
struct intrq {
TAILQ_HEAD(, intrhand) iq_list; /* handler list */
struct cpu_info *iq_ci; /* CPU the IRQ runs on */
int iq_irq_max; /* IRQ to mask while handling */
int iq_irq_min; /* lowest IRQ when shared */
int iq_ist; /* share type */
int iq_route;
};
struct agintc_dmamem {
bus_dmamap_t adm_map;
bus_dma_segment_t adm_seg;
size_t adm_size;
caddr_t adm_kva;
};
#define AGINTC_DMA_MAP(_adm) ((_adm)->adm_map)
#define AGINTC_DMA_LEN(_adm) ((_adm)->adm_size)
#define AGINTC_DMA_DVA(_adm) ((_adm)->adm_map->dm_segs[0].ds_addr)
#define AGINTC_DMA_KVA(_adm) ((void *)(_adm)->adm_kva)
struct agintc_dmamem *agintc_dmamem_alloc(bus_dma_tag_t, bus_size_t,
bus_size_t);
void agintc_dmamem_free(bus_dma_tag_t, struct agintc_dmamem *);
int agintc_match(struct device *, void *, void *);
void agintc_attach(struct device *, struct device *, void *);
void agintc_mbiinit(struct agintc_softc *, int, bus_addr_t);
void agintc_cpuinit(void);
int agintc_spllower(int);
void agintc_splx(int);
int agintc_splraise(int);
void agintc_setipl(int);
void agintc_enable_wakeup(void);
void agintc_disable_wakeup(void);
void agintc_calc_mask(void);
void agintc_calc_irq(struct agintc_softc *sc, int irq);
void *agintc_intr_establish(int, int, int, struct cpu_info *,
int (*)(void *), void *, char *);
void *agintc_intr_establish_fdt(void *cookie, int *cell, int level,
struct cpu_info *, int (*func)(void *), void *arg, char *name);
void *agintc_intr_establish_mbi(void *, uint64_t *, uint64_t *,
int , struct cpu_info *, int (*)(void *), void *, char *);
void agintc_intr_disestablish(void *);
void agintc_intr_set_wakeup(void *);
void agintc_irq_handler(void *);
uint32_t agintc_iack(void);
void agintc_eoi(uint32_t);
void agintc_set_priority(struct agintc_softc *sc, int, int);
void agintc_intr_enable(struct agintc_softc *, int);
void agintc_intr_disable(struct agintc_softc *, int);
void agintc_intr_config(struct agintc_softc *, int, int);
void agintc_route(struct agintc_softc *, int, int,
struct cpu_info *);
void agintc_route_irq(void *, int, struct cpu_info *);
void agintc_intr_barrier(void *);
void agintc_r_wait_rwp(struct agintc_softc *sc);
int agintc_ipi_ddb(void *v);
int agintc_ipi_halt(void *v);
int agintc_ipi_nop(void *v);
int agintc_ipi_combined(void *);
void agintc_send_ipi(struct cpu_info *, int);
void agintc_msi_discard(struct agintc_lpi_info *);
void agintc_msi_inv(struct agintc_lpi_info *);
const struct cfattach agintc_ca = {
sizeof (struct agintc_softc), agintc_match, agintc_attach
};
struct cfdriver agintc_cd = {
NULL, "agintc", DV_DULL
};
static char *agintc_compatibles[] = {
"arm,gic-v3",
"arm,gic-v4",
NULL
};
int
agintc_match(struct device *parent, void *cfdata, void *aux)
{
struct fdt_attach_args *faa = aux;
int i;
for (i = 0; agintc_compatibles[i]; i++)
if (OF_is_compatible(faa->fa_node, agintc_compatibles[i]))
return (1);
return (0);
}
static void
__isb(void)
{
__asm volatile("isb");
}
void
agintc_attach(struct device *parent, struct device *self, void *aux)
{
struct agintc_softc *sc = (struct agintc_softc *)self;
struct fdt_attach_args *faa = aux;
struct cpu_info *ci;
CPU_INFO_ITERATOR cii;
u_long psw;
uint32_t typer;
uint32_t nsacr, oldnsacr;
uint32_t pmr, oldpmr;
uint32_t ctrl, bits;
uint32_t affinity;
uint64_t redist_stride;
int i, nbits, nintr;
int offset, nredist;
#ifdef MULTIPROCESSOR
int nipi, ipiirq[3];
#endif
psw = intr_disable();
arm_init_smask();
sc->sc_iot = faa->fa_iot;
sc->sc_dmat = faa->fa_dmat;
/* First row: distributor */
if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr,
faa->fa_reg[0].size, 0, &sc->sc_d_ioh))
panic("%s: ICD bus_space_map failed!", __func__);
/* Second row: redistributor */
if (bus_space_map(sc->sc_iot, faa->fa_reg[1].addr,
faa->fa_reg[1].size, 0, &sc->sc_redist_base))
panic("%s: ICP bus_space_map failed!", __func__);
typer = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_TYPER);
if (typer & GICD_TYPER_LPIS) {
/* Allocate redistributor tables */
sc->sc_prop = agintc_dmamem_alloc(sc->sc_dmat,
GICR_PROP_SIZE, GICR_PROP_SIZE);
if (sc->sc_prop == NULL) {
printf(": can't alloc LPI config table\n");
goto unmap;
}
sc->sc_pend = agintc_dmamem_alloc(sc->sc_dmat,
GICR_PEND_SIZE, GICR_PEND_SIZE);
if (sc->sc_pend == NULL) {
printf(": can't alloc LPI pending table\n");
goto unmap;
}
/* Minimum number of LPIs supported by any implementation. */
sc->sc_nlpi = 8192;
}
if (typer & GICD_TYPER_MBIS)
agintc_mbiinit(sc, faa->fa_node, faa->fa_reg[0].addr);
/*
* We are guaranteed to have at least 16 priority levels, so
* in principle we just want to use the top 4 bits of the
* (non-secure) priority field.
*/
sc->sc_prio_shift = sc->sc_pmr_shift = 4;
/*
* If the system supports two security states and SCR_EL3.FIQ
* is zero, the non-secure shifted view applies. We detect
* this by checking whether the number of writable bits
* matches the number of implemented priority bits. If that
* is the case we will need to adjust the priorities that we
* write into ICC_PMR_EL1 accordingly.
*
* On Ampere eMAG it appears as if there are five writable
* bits when we write 0xff. But for higher priorities
* (smaller values) only the top 4 bits stick. So we use 0xbf
* instead to determine the number of writable bits.
*/
ctrl = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR);
if ((ctrl & GICD_CTLR_DS) == 0) {
__asm volatile("mrs %x0, "STR(ICC_CTLR_EL1) : "=r"(ctrl));
nbits = ICC_CTLR_EL1_PRIBITS(ctrl) + 1;
__asm volatile("mrs %x0, "STR(ICC_PMR) : "=r"(oldpmr));
__asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xbf));
__asm volatile("mrs %x0, "STR(ICC_PMR) : "=r"(pmr));
__asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(oldpmr));
if (nbits == 8 - (ffs(pmr) - 1))
sc->sc_pmr_shift--;
}
/*
* The Rockchip RK3399 is busted. Its GIC-500 treats all
* access to its memory mapped registers as "secure". As a
* result, several registers don't behave as expected. For
* example, the GICD_IPRIORITYRn and GICR_IPRIORITYRn
* registers expose the full priority range available to
* secure interrupts. We need to be aware of this and write
* an adjusted priority value into these registers. We also
* need to be careful not to touch any bits that shouldn't be
* writable in non-secure mode.
*
* We check whether we have secure mode access to these
* registers by attempting to write to the GICD_NSACR register
* and check whether its contents actually change. In that
* case we need to adjust the priorities we write into
* GICD_IPRIORITYRn and GICRIPRIORITYRn accordingly.
*/
oldnsacr = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32));
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32),
oldnsacr ^ 0xffffffff);
nsacr = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32));
if (nsacr != oldnsacr) {
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32),
oldnsacr);
sc->sc_rk3399_quirk = 1;
sc->sc_prio_shift--;
printf(" sec");
}
printf(" shift %d:%d", sc->sc_prio_shift, sc->sc_pmr_shift);
evcount_attach(&sc->sc_spur, "irq1023/spur", NULL);
__asm volatile("msr "STR(ICC_SRE_EL1)", %x0" : : "r" (ICC_SRE_EL1_EN));
__isb();
nintr = 32 * (typer & GICD_TYPER_ITLINE_M);
nintr += 32; /* ICD_ICTR + 1, irq 0-31 is SGI, 32+ is PPI */
sc->sc_nintr = nintr;
agintc_sc = sc; /* save this for global access */
/* find the redistributors. */
offset = 0;
redist_stride = OF_getpropint64(faa->fa_node, "redistributor-stride", 0);
for (nredist = 0; ; nredist++) {
uint64_t typer;
int32_t sz;
typer = bus_space_read_8(sc->sc_iot, sc->sc_redist_base,
offset + GICR_TYPER);
if (redist_stride == 0) {
sz = (64 * 1024 * 2);
if (typer & GICR_TYPER_VLPIS)
sz += (64 * 1024 * 2);
} else
sz = redist_stride;
#ifdef DEBUG_AGINTC
printf("probing redistributor %d %x\n", nredist, offset);
#endif
offset += sz;
if (typer & GICR_TYPER_LAST) {
sc->sc_num_redist = nredist + 1;
break;
}
}
printf(" nirq %d nredist %d", nintr, sc->sc_num_redist);
sc->sc_r_ioh = mallocarray(sc->sc_num_redist,
sizeof(*sc->sc_r_ioh), M_DEVBUF, M_WAITOK);
sc->sc_processor = mallocarray(sc->sc_num_redist,
sizeof(*sc->sc_processor), M_DEVBUF, M_WAITOK);
/* submap and configure the redistributors. */
offset = 0;
for (nredist = 0; nredist < sc->sc_num_redist; nredist++) {
uint64_t typer;
int32_t sz;
typer = bus_space_read_8(sc->sc_iot, sc->sc_redist_base,
offset + GICR_TYPER);
if (redist_stride == 0) {
sz = (64 * 1024 * 2);
if (typer & GICR_TYPER_VLPIS)
sz += (64 * 1024 * 2);
} else
sz = redist_stride;
affinity = bus_space_read_8(sc->sc_iot,
sc->sc_redist_base, offset + GICR_TYPER) >> 32;
CPU_INFO_FOREACH(cii, ci) {
if (affinity == (((ci->ci_mpidr >> 8) & 0xff000000) |
(ci->ci_mpidr & 0x00ffffff)))
break;
}
if (ci != NULL)
sc->sc_cpuremap[ci->ci_cpuid] = nredist;
sc->sc_processor[nredist] = bus_space_read_8(sc->sc_iot,
sc->sc_redist_base, offset + GICR_TYPER) >> 8;
bus_space_subregion(sc->sc_iot, sc->sc_redist_base,
offset, sz, &sc->sc_r_ioh[nredist]);
if (sc->sc_nlpi > 0) {
bus_space_write_8(sc->sc_iot, sc->sc_redist_base,
offset + GICR_PROPBASER,
AGINTC_DMA_DVA(sc->sc_prop) |
GICR_PROPBASER_ISH | GICR_PROPBASER_IC_NORM_NC |
fls(LPI_BASE + sc->sc_nlpi - 1) - 1);
bus_space_write_8(sc->sc_iot, sc->sc_redist_base,
offset + GICR_PENDBASER,
AGINTC_DMA_DVA(sc->sc_pend) |
GICR_PENDBASER_ISH | GICR_PENDBASER_IC_NORM_NC |
GICR_PENDBASER_PTZ);
bus_space_write_4(sc->sc_iot, sc->sc_redist_base,
offset + GICR_CTLR, GICR_CTLR_ENABLE_LPIS);
}
offset += sz;
}
/* Disable all interrupts, clear all pending */
for (i = 1; i < nintr / 32; i++) {
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh,
GICD_ICACTIVER(i * 32), ~0);
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh,
GICD_ICENABLER(i * 32), ~0);
}
for (i = 4; i < nintr; i += 4) {
/* lowest priority ?? */
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh,
GICD_IPRIORITYR(i), 0xffffffff);
}
/* Set all interrupts to G1NS */
for (i = 1; i < nintr / 32; i++) {
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh,
GICD_IGROUPR(i * 32), ~0);
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh,
GICD_IGRPMODR(i * 32), 0);
}
for (i = 2; i < nintr / 16; i++) {
/* irq 32 - N */
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh,
GICD_ICFGR(i * 16), 0);
}
agintc_cpuinit();
sc->sc_handler = mallocarray(nintr,
sizeof(*sc->sc_handler), M_DEVBUF, M_ZERO | M_WAITOK);
for (i = 0; i < nintr; i++)
TAILQ_INIT(&sc->sc_handler[i].iq_list);
sc->sc_lpi = mallocarray(sc->sc_nlpi,
sizeof(*sc->sc_lpi), M_DEVBUF, M_ZERO | M_WAITOK);
/* set priority to IPL_HIGH until configure lowers to desired IPL */
agintc_setipl(IPL_HIGH);
/* initialize all interrupts as disabled */
agintc_calc_mask();
/* insert self as interrupt handler */
arm_set_intr_handler(agintc_splraise, agintc_spllower, agintc_splx,
agintc_setipl, agintc_irq_handler, NULL,
agintc_enable_wakeup, agintc_disable_wakeup);
/* enable interrupts */
ctrl = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR);
bits = GICD_CTLR_ARE_NS | GICD_CTLR_EnableGrp1A | GICD_CTLR_EnableGrp1;
if (sc->sc_rk3399_quirk) {
bits &= ~GICD_CTLR_EnableGrp1A;
bits <<= 1;
}
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR, ctrl | bits);
__asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xff));
__asm volatile("msr "STR(ICC_BPR1)", %x0" :: "r"(0));
__asm volatile("msr "STR(ICC_IGRPEN1)", %x0" :: "r"(1));
#ifdef MULTIPROCESSOR
/* setup IPI interrupts */
/*
* Ideally we want three IPI interrupts, one for NOP, one for
* DDB and one for HALT. However we can survive if only one
* is available; it is possible that most are not available to
* the non-secure OS.
*/
nipi = 0;
for (i = 0; i < 16; i++) {
int hwcpu = sc->sc_cpuremap[cpu_number()];
int reg, oldreg;
oldreg = bus_space_read_1(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_IPRIORITYR(i));
bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_IPRIORITYR(i), oldreg ^ 0x20);
/* if this interrupt is not usable, pri will be unmodified */
reg = bus_space_read_1(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_IPRIORITYR(i));
if (reg == oldreg)
continue;
/* return to original value, will be set when used */
bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_IPRIORITYR(i), oldreg);
if (nipi == 0)
printf(" ipi: %d", i);
else
printf(", %d", i);
ipiirq[nipi++] = i;
if (nipi == 3)
break;
}
if (nipi == 0)
panic("no irq available for IPI");
switch (nipi) {
case 1:
sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0],
IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL,
agintc_ipi_combined, sc, "ipi");
sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0];
sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[0];
sc->sc_ipi_num[ARM_IPI_HALT] = ipiirq[0];
break;
case 2:
sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0],
IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL,
agintc_ipi_nop, sc, "ipinop");
sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0];
sc->sc_ipi_irq[1] = agintc_intr_establish(ipiirq[1],
IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL,
agintc_ipi_combined, sc, "ipi");
sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[1];
sc->sc_ipi_num[ARM_IPI_HALT] = ipiirq[1];
break;
case 3:
sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0],
IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL,
agintc_ipi_nop, sc, "ipinop");
sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0];
sc->sc_ipi_irq[1] = agintc_intr_establish(ipiirq[1],
IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL,
agintc_ipi_ddb, sc, "ipiddb");
sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[1];
sc->sc_ipi_irq[2] = agintc_intr_establish(ipiirq[2],
IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL,
agintc_ipi_halt, sc, "ipihalt");
sc->sc_ipi_num[ARM_IPI_HALT] = ipiirq[2];
break;
default:
panic("nipi unexpected number %d", nipi);
}
intr_send_ipi_func = agintc_send_ipi;
#endif
sc->sc_ic.ic_node = faa->fa_node;
sc->sc_ic.ic_cookie = self;
sc->sc_ic.ic_establish = agintc_intr_establish_fdt;
sc->sc_ic.ic_disestablish = agintc_intr_disestablish;
sc->sc_ic.ic_route = agintc_route_irq;
sc->sc_ic.ic_cpu_enable = agintc_cpuinit;
sc->sc_ic.ic_barrier = agintc_intr_barrier;
if (sc->sc_mbi_nranges > 0)
sc->sc_ic.ic_establish_msi = agintc_intr_establish_mbi;
sc->sc_ic.ic_set_wakeup = agintc_intr_set_wakeup;
arm_intr_register_fdt(&sc->sc_ic);
intr_restore(psw);
/* Attach ITS. */
simplebus_attach(parent, &sc->sc_sbus.sc_dev, faa);
return;
unmap:
if (sc->sc_r_ioh) {
free(sc->sc_r_ioh, M_DEVBUF,
sc->sc_num_redist * sizeof(*sc->sc_r_ioh));
}
if (sc->sc_processor) {
free(sc->sc_processor, M_DEVBUF,
sc->sc_num_redist * sizeof(*sc->sc_processor));
}
if (sc->sc_pend)
agintc_dmamem_free(sc->sc_dmat, sc->sc_pend);
if (sc->sc_prop)
agintc_dmamem_free(sc->sc_dmat, sc->sc_prop);
bus_space_unmap(sc->sc_iot, sc->sc_redist_base, faa->fa_reg[1].size);
bus_space_unmap(sc->sc_iot, sc->sc_d_ioh, faa->fa_reg[0].size);
}
void
agintc_mbiinit(struct agintc_softc *sc, int node, bus_addr_t addr)
{
uint32_t *ranges;
int i, len;
if (OF_getproplen(node, "msi-controller") != 0)
return;
len = OF_getproplen(node, "mbi-ranges");
if (len <= 0 || len % 2 * sizeof(uint32_t) != 0)
return;
ranges = malloc(len, M_TEMP, M_WAITOK);
OF_getpropintarray(node, "mbi-ranges", ranges, len);
sc->sc_mbi_nranges = len / (2 * sizeof(uint32_t));
sc->sc_mbi_ranges = mallocarray(sc->sc_mbi_nranges,
sizeof(struct agintc_mbi_range), M_DEVBUF, M_WAITOK);
for (i = 0; i < sc->sc_mbi_nranges; i++) {
sc->sc_mbi_ranges[i].mr_base = ranges[2 * i + 0];
sc->sc_mbi_ranges[i].mr_span = ranges[2 * i + 1];
sc->sc_mbi_ranges[i].mr_mbi =
mallocarray(sc->sc_mbi_ranges[i].mr_span,
sizeof(void *), M_DEVBUF, M_WAITOK | M_ZERO);
}
free(ranges, M_TEMP, len);
addr = OF_getpropint64(node, "mbi-alias", addr);
sc->sc_mbi_addr = addr + GICD_SETSPI_NSR;
printf(" mbi");
}
/* Initialize redistributors on each core. */
void
agintc_cpuinit(void)
{
struct agintc_softc *sc = agintc_sc;
uint32_t waker;
int timeout = 100000;
int hwcpu;
int i;
hwcpu = sc->sc_cpuremap[cpu_number()];
waker = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_WAKER);
waker &= ~(GICR_WAKER_PROCESSORSLEEP);
bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], GICR_WAKER,
waker);
do {
waker = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_WAKER);
} while (--timeout && (waker & GICR_WAKER_CHILDRENASLEEP));
if (timeout == 0)
printf("%s: waker timed out\n", __func__);
bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_ICENABLE0, ~0);
bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_ICPENDR0, ~0);
bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_ICACTIVE0, ~0);
for (i = 0; i < 32; i += 4) {
bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_IPRIORITYR(i), ~0);
}
bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_IGROUPR0, ~0);
bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_IGRPMODR0, 0);
if (sc->sc_ipi_irq[0] != NULL)
agintc_route_irq(sc->sc_ipi_irq[0], IRQ_ENABLE, curcpu());
if (sc->sc_ipi_irq[1] != NULL)
agintc_route_irq(sc->sc_ipi_irq[1], IRQ_ENABLE, curcpu());
if (sc->sc_ipi_irq[2] != NULL)
agintc_route_irq(sc->sc_ipi_irq[2], IRQ_ENABLE, curcpu());
__asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xff));
__asm volatile("msr "STR(ICC_BPR1)", %x0" :: "r"(0));
__asm volatile("msr "STR(ICC_IGRPEN1)", %x0" :: "r"(1));
intr_enable();
}
void
agintc_set_priority(struct agintc_softc *sc, int irq, int ipl)
{
struct cpu_info *ci = curcpu();
int hwcpu = sc->sc_cpuremap[ci->ci_cpuid];
uint32_t prival;
prival = ((0xff - ipl) << sc->sc_prio_shift) & 0xff;
if (irq >= SPI_BASE) {
bus_space_write_1(sc->sc_iot, sc->sc_d_ioh,
GICD_IPRIORITYR(irq), prival);
} else {
/* only sets local redistributor */
bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_IPRIORITYR(irq), prival);
}
}
void
agintc_setipl(int ipl)
{
struct agintc_softc *sc = agintc_sc;
struct cpu_info *ci = curcpu();
u_long psw;
uint32_t prival;
/* disable here is only to keep hardware in sync with ci->ci_cpl */
psw = intr_disable();
ci->ci_cpl = ipl;
prival = ((0xff - ipl) << sc->sc_pmr_shift) & 0xff;
__asm volatile("msr "STR(ICC_PMR)", %x0" : : "r" (prival));
__isb();
intr_restore(psw);
}
void
agintc_enable_wakeup(void)
{
struct agintc_softc *sc = agintc_sc;
struct intrhand *ih;
uint8_t *prop;
int irq, wakeup;
for (irq = 0; irq < sc->sc_nintr; irq++) {
/* No handler? Disabled already. */
if (TAILQ_EMPTY(&sc->sc_handler[irq].iq_list))
continue;
/* Unless we're WAKEUP, disable. */
wakeup = 0;
TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) {
if (ih->ih_flags & IPL_WAKEUP) {
wakeup = 1;
break;
}
}
if (!wakeup)
agintc_intr_disable(sc, irq);
}
for (irq = 0; irq < sc->sc_nlpi; irq++) {
if (sc->sc_lpi[irq] == NULL)
continue;
ih = sc->sc_lpi[irq]->li_ih;
KASSERT(ih != NULL);
if (ih->ih_flags & IPL_WAKEUP)
continue;
prop = AGINTC_DMA_KVA(sc->sc_prop);
prop[irq] &= ~GICR_PROP_ENABLE;
/* Make globally visible. */
cpu_dcache_wb_range((vaddr_t)&prop[irq],
sizeof(*prop));
__asm volatile("dsb sy");
/* Invalidate cache */
agintc_msi_inv(sc->sc_lpi[irq]);
}
}
void
agintc_disable_wakeup(void)
{
struct agintc_softc *sc = agintc_sc;
struct intrhand *ih;
uint8_t *prop;
int irq, wakeup;
for (irq = 0; irq < sc->sc_nintr; irq++) {
/* No handler? Keep disabled. */
if (TAILQ_EMPTY(&sc->sc_handler[irq].iq_list))
continue;
/* WAKEUPs are already enabled. */
wakeup = 0;
TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) {
if (ih->ih_flags & IPL_WAKEUP) {
wakeup = 1;
break;
}
}
if (!wakeup)
agintc_intr_enable(sc, irq);
}
for (irq = 0; irq < sc->sc_nlpi; irq++) {
if (sc->sc_lpi[irq] == NULL)
continue;
ih = sc->sc_lpi[irq]->li_ih;
KASSERT(ih != NULL);
if (ih->ih_flags & IPL_WAKEUP)
continue;
prop = AGINTC_DMA_KVA(sc->sc_prop);
prop[irq] |= GICR_PROP_ENABLE;
/* Make globally visible. */
cpu_dcache_wb_range((vaddr_t)&prop[irq],
sizeof(*prop));
__asm volatile("dsb sy");
/* Invalidate cache */
agintc_msi_inv(sc->sc_lpi[irq]);
}
}
void
agintc_intr_enable(struct agintc_softc *sc, int irq)
{
struct cpu_info *ci = curcpu();
int hwcpu = sc->sc_cpuremap[ci->ci_cpuid];
int bit = 1 << IRQ_TO_REG32BIT(irq);
if (irq >= 32) {
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh,
GICD_ISENABLER(irq), bit);
} else {
bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_ISENABLE0, bit);
}
}
void
agintc_intr_disable(struct agintc_softc *sc, int irq)
{
struct cpu_info *ci = curcpu();
int hwcpu = sc->sc_cpuremap[ci->ci_cpuid];
if (irq >= 32) {
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh,
GICD_ICENABLER(irq), 1 << IRQ_TO_REG32BIT(irq));
} else {
bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_ICENABLE0, 1 << IRQ_TO_REG32BIT(irq));
}
}
void
agintc_intr_config(struct agintc_softc *sc, int irq, int type)
{
uint32_t reg;
/* Don't dare to change SGIs or PPIs (yet) */
if (irq < 32)
return;
reg = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_ICFGR(irq));
reg &= ~GICD_ICFGR_TRIG_MASK(irq);
if (type == IST_EDGE_RISING)
reg |= GICD_ICFGR_TRIG_EDGE(irq);
else
reg |= GICD_ICFGR_TRIG_LEVEL(irq);
bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_ICFGR(irq), reg);
}
void
agintc_calc_mask(void)
{
struct agintc_softc *sc = agintc_sc;
int irq;
for (irq = 0; irq < sc->sc_nintr; irq++)
agintc_calc_irq(sc, irq);
}
void
agintc_calc_irq(struct agintc_softc *sc, int irq)
{
struct cpu_info *ci = sc->sc_handler[irq].iq_ci;
struct intrhand *ih;
int max = IPL_NONE;
int min = IPL_HIGH;
TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) {
if (ih->ih_ipl > max)
max = ih->ih_ipl;
if (ih->ih_ipl < min)
min = ih->ih_ipl;
}
if (max == IPL_NONE)
min = IPL_NONE;
if (sc->sc_handler[irq].iq_irq_max == max &&
sc->sc_handler[irq].iq_irq_min == min)
return;
sc->sc_handler[irq].iq_irq_max = max;
sc->sc_handler[irq].iq_irq_min = min;
#ifdef DEBUG_AGINTC
if (min != IPL_NONE)
printf("irq %d to block at %d %d \n", irq, max, min );
#endif
/* Enable interrupts at lower levels, clear -> enable */
/* Set interrupt priority/enable */
if (min != IPL_NONE) {
agintc_set_priority(sc, irq, min);
agintc_route(sc, irq, IRQ_ENABLE, ci);
agintc_intr_enable(sc, irq);
} else {
agintc_intr_disable(sc, irq);
agintc_route(sc, irq, IRQ_DISABLE, ci);
}
}
void
agintc_splx(int new)
{
struct cpu_info *ci = curcpu();
if (ci->ci_ipending & arm_smask[new])
arm_do_pending_intr(new);
agintc_setipl(new);
}
int
agintc_spllower(int new)
{
struct cpu_info *ci = curcpu();
int old = ci->ci_cpl;
agintc_splx(new);
return (old);
}
int
agintc_splraise(int new)
{
struct cpu_info *ci = curcpu();
int old = ci->ci_cpl;
/*
* setipl must always be called because there is a race window
* where the variable is updated before the mask is set
* an interrupt occurs in that window without the mask always
* being set, the hardware might not get updated on the next
* splraise completely messing up spl protection.
*/
if (old > new)
new = old;
agintc_setipl(new);
return (old);
}
uint32_t
agintc_iack(void)
{
int irq;
__asm volatile("mrs %x0, "STR(ICC_IAR1) : "=r" (irq));
__asm volatile("dsb sy");
return irq;
}
void
agintc_route_irq(void *v, int enable, struct cpu_info *ci)
{
struct agintc_softc *sc = agintc_sc;
struct intrhand *ih = v;
if (enable) {
agintc_set_priority(sc, ih->ih_irq,
sc->sc_handler[ih->ih_irq].iq_irq_min);
agintc_route(sc, ih->ih_irq, IRQ_ENABLE, ci);
agintc_intr_enable(sc, ih->ih_irq);
}
}
void
agintc_route(struct agintc_softc *sc, int irq, int enable, struct cpu_info *ci)
{
/* XXX does not yet support 'participating node' */
if (irq >= 32) {
#ifdef DEBUG_AGINTC
printf("router %x irq %d val %016llx\n", GICD_IROUTER(irq),
irq, ci->ci_mpidr & MPIDR_AFF);
#endif
bus_space_write_8(sc->sc_iot, sc->sc_d_ioh,
GICD_IROUTER(irq), ci->ci_mpidr & MPIDR_AFF);
}
}
void
agintc_intr_barrier(void *cookie)
{
struct intrhand *ih = cookie;
sched_barrier(ih->ih_ci);
}
void
agintc_run_handler(struct intrhand *ih, void *frame, int s)
{
void *arg;
int handled;
#ifdef MULTIPROCESSOR
int need_lock;
if (ih->ih_flags & IPL_MPSAFE)
need_lock = 0;
else
need_lock = s < IPL_SCHED;
if (need_lock)
KERNEL_LOCK();
#endif
if (ih->ih_arg)
arg = ih->ih_arg;
else
arg = frame;
handled = ih->ih_func(arg);
if (handled)
ih->ih_count.ec_count++;
#ifdef MULTIPROCESSOR
if (need_lock)
KERNEL_UNLOCK();
#endif
}
void
agintc_irq_handler(void *frame)
{
struct agintc_softc *sc = agintc_sc;
struct intrhand *ih;
int irq, pri, s;
irq = agintc_iack();
#ifdef DEBUG_AGINTC
if (irq != 30)
printf("irq %d fired\n", irq);
else {
static int cnt = 0;
if ((cnt++ % 100) == 0) {
printf("irq %d fired * _100\n", irq);
#ifdef DDB
db_enter();
#endif
}
}
#endif
if (irq == 1023) {
sc->sc_spur.ec_count++;
return;
}
if ((irq >= sc->sc_nintr && irq < LPI_BASE) ||
irq >= LPI_BASE + sc->sc_nlpi) {
return;
}
if (irq >= LPI_BASE) {
if (sc->sc_lpi[irq - LPI_BASE] == NULL)
return;
ih = sc->sc_lpi[irq - LPI_BASE]->li_ih;
KASSERT(ih != NULL);
s = agintc_splraise(ih->ih_ipl);
intr_enable();
agintc_run_handler(ih, frame, s);
intr_disable();
agintc_eoi(irq);
agintc_splx(s);
return;
}
pri = sc->sc_handler[irq].iq_irq_max;
s = agintc_splraise(pri);
intr_enable();
TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) {
agintc_run_handler(ih, frame, s);
}
intr_disable();
agintc_eoi(irq);
agintc_splx(s);
}
void *
agintc_intr_establish_fdt(void *cookie, int *cell, int level,
struct cpu_info *ci, int (*func)(void *), void *arg, char *name)
{
struct agintc_softc *sc = agintc_sc;
int irq;
int type;
/* 2nd cell contains the interrupt number */
irq = cell[1];
/* 1st cell contains type: 0 SPI (32-X), 1 PPI (16-31) */
if (cell[0] == 0)
irq += SPI_BASE;
else if (cell[0] == 1)
irq += PPI_BASE;
else
panic("%s: bogus interrupt type", sc->sc_sbus.sc_dev.dv_xname);
/* SPIs are only active-high level or low-to-high edge */
if (cell[2] & 0x3)
type = IST_EDGE_RISING;
else
type = IST_LEVEL_HIGH;
return agintc_intr_establish(irq, type, level, ci, func, arg, name);
}
void *
agintc_intr_establish(int irqno, int type, int level, struct cpu_info *ci,
int (*func)(void *), void *arg, char *name)
{
struct agintc_softc *sc = agintc_sc;
struct intrhand *ih;
u_long psw;
if (irqno < 0 || (irqno >= sc->sc_nintr && irqno < LPI_BASE) ||
irqno >= LPI_BASE + sc->sc_nlpi)
panic("agintc_intr_establish: bogus irqnumber %d: %s",
irqno, name);
if (ci == NULL)
ci = &cpu_info_primary;
ih = malloc(sizeof *ih, M_DEVBUF, M_WAITOK);
ih->ih_func = func;
ih->ih_arg = arg;
ih->ih_ipl = level & IPL_IRQMASK;
ih->ih_flags = level & IPL_FLAGMASK;
ih->ih_irq = irqno;
ih->ih_name = name;
ih->ih_ci = ci;
psw = intr_disable();
if (irqno < LPI_BASE) {
if (!TAILQ_EMPTY(&sc->sc_handler[irqno].iq_list) &&
sc->sc_handler[irqno].iq_ci != ci) {
intr_restore(psw);
free(ih, M_DEVBUF, sizeof *ih);
return NULL;
}
TAILQ_INSERT_TAIL(&sc->sc_handler[irqno].iq_list, ih, ih_list);
sc->sc_handler[irqno].iq_ci = ci;
}
if (name != NULL)
evcount_attach(&ih->ih_count, name, &ih->ih_irq);
#ifdef DEBUG_AGINTC
printf("%s: irq %d level %d [%s]\n", __func__, irqno, level, name);
#endif
if (irqno < LPI_BASE) {
agintc_intr_config(sc, irqno, type);
agintc_calc_irq(sc, irqno);
} else {
uint8_t *prop = AGINTC_DMA_KVA(sc->sc_prop);
prop[irqno - LPI_BASE] = (((0xff - ih->ih_ipl) << 4) & 0xff) |
GICR_PROP_GROUP1 | GICR_PROP_ENABLE;
/* Make globally visible. */
cpu_dcache_wb_range((vaddr_t)&prop[irqno - LPI_BASE],
sizeof(*prop));
__asm volatile("dsb sy");
}
intr_restore(psw);
return (ih);
}
void
agintc_intr_disestablish(void *cookie)
{
struct agintc_softc *sc = agintc_sc;
struct intrhand *ih = cookie;
int irqno = ih->ih_irq;
u_long psw;
struct agintc_mbi_range *mr;
int i;
psw = intr_disable();
if (irqno < LPI_BASE) {
TAILQ_REMOVE(&sc->sc_handler[irqno].iq_list, ih, ih_list);
agintc_calc_irq(sc, irqno);
/* In case this is an MBI, free it */
for (i = 0; i < sc->sc_mbi_nranges; i++) {
mr = &sc->sc_mbi_ranges[i];
if (irqno < mr->mr_base)
continue;
if (irqno >= mr->mr_base + mr->mr_span)
break;
if (mr->mr_mbi[irqno - mr->mr_base] != NULL)
mr->mr_mbi[irqno - mr->mr_base] = NULL;
}
} else {
uint8_t *prop = AGINTC_DMA_KVA(sc->sc_prop);
prop[irqno - LPI_BASE] = 0;
/* Make globally visible. */
cpu_dcache_wb_range((vaddr_t)&prop[irqno - LPI_BASE],
sizeof(*prop));
__asm volatile("dsb sy");
}
if (ih->ih_name != NULL)
evcount_detach(&ih->ih_count);
intr_restore(psw);
free(ih, M_DEVBUF, 0);
}
void
agintc_intr_set_wakeup(void *cookie)
{
struct intrhand *ih = cookie;
ih->ih_flags |= IPL_WAKEUP;
}
void *
agintc_intr_establish_mbi(void *self, uint64_t *addr, uint64_t *data,
int level, struct cpu_info *ci, int (*func)(void *), void *arg, char *name)
{
struct agintc_softc *sc = agintc_sc;
struct agintc_mbi_range *mr;
void *cookie;
int i, j, hwcpu;
if (ci == NULL)
ci = &cpu_info_primary;
hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid];
for (i = 0; i < sc->sc_mbi_nranges; i++) {
mr = &sc->sc_mbi_ranges[i];
for (j = 0; j < mr->mr_span; j++) {
if (mr->mr_mbi[j] != NULL)
continue;
cookie = agintc_intr_establish(mr->mr_base + j,
IST_EDGE_RISING, level, ci, func, arg, name);
if (cookie == NULL)
return NULL;
*addr = sc->sc_mbi_addr;
*data = mr->mr_base + j;
mr->mr_mbi[j] = cookie;
return cookie;
}
}
return NULL;
}
void
agintc_eoi(uint32_t eoi)
{
__asm volatile("msr "STR(ICC_EOIR1)", %x0" :: "r" (eoi));
__isb();
}
void
agintc_d_wait_rwp(struct agintc_softc *sc)
{
int count = 100000;
uint32_t v;
do {
v = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR);
} while (--count && (v & GICD_CTLR_RWP));
if (count == 0)
panic("%s: RWP timed out 0x08%x", __func__, v);
}
void
agintc_r_wait_rwp(struct agintc_softc *sc)
{
struct cpu_info *ci = curcpu();
int hwcpu = sc->sc_cpuremap[ci->ci_cpuid];
int count = 100000;
uint32_t v;
do {
v = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu],
GICR_CTLR);
} while (--count && (v & GICR_CTLR_RWP));
if (count == 0)
panic("%s: RWP timed out 0x08%x", __func__, v);
}
#ifdef MULTIPROCESSOR
int
agintc_ipi_ddb(void *v)
{
/* XXX */
#ifdef DDB
db_enter();
#endif
return 1;
}
int
agintc_ipi_halt(void *v)
{
struct agintc_softc *sc = v;
int old = curcpu()->ci_cpl;
intr_disable();
agintc_eoi(sc->sc_ipi_num[ARM_IPI_HALT]);
agintc_setipl(IPL_NONE);
cpu_halt();
agintc_setipl(old);
intr_enable();
return 1;
}
int
agintc_ipi_nop(void *v)
{
/* Nothing to do here, just enough to wake up from WFI */
return 1;
}
int
agintc_ipi_combined(void *v)
{
struct agintc_softc *sc = v;
if (sc->sc_ipi_reason[cpu_number()] == ARM_IPI_DDB) {
sc->sc_ipi_reason[cpu_number()] = ARM_IPI_NOP;
return agintc_ipi_ddb(v);
} else if (sc->sc_ipi_reason[cpu_number()] == ARM_IPI_HALT) {
sc->sc_ipi_reason[cpu_number()] = ARM_IPI_NOP;
return agintc_ipi_halt(v);
} else {
return agintc_ipi_nop(v);
}
}
void
agintc_send_ipi(struct cpu_info *ci, int id)
{
struct agintc_softc *sc = agintc_sc;
uint64_t sendmask;
if (ci == curcpu() && id == ARM_IPI_NOP)
return;
/* never overwrite IPI_DDB or IPI_HALT with IPI_NOP */
if (id == ARM_IPI_DDB || id == ARM_IPI_HALT)
sc->sc_ipi_reason[ci->ci_cpuid] = id;
/* will only send 1 cpu */
sendmask = (ci->ci_mpidr & MPIDR_AFF3) << 16;
sendmask |= (ci->ci_mpidr & MPIDR_AFF2) << 16;
sendmask |= (ci->ci_mpidr & MPIDR_AFF1) << 8;
sendmask |= 1 << (ci->ci_mpidr & 0x0f);
sendmask |= (sc->sc_ipi_num[id] << 24);
__asm volatile ("msr " STR(ICC_SGI1R)", %x0" ::"r"(sendmask));
}
#endif
/*
* GICv3 ITS controller for MSI interrupts.
*/
#define GITS_CTLR 0x0000
#define GITS_CTLR_ENABLED (1UL << 0)
#define GITS_TYPER 0x0008
#define GITS_TYPER_CIL (1ULL << 36)
#define GITS_TYPER_CIDBITS(x) (((x) >> 32) & 0xf)
#define GITS_TYPER_HCC(x) (((x) >> 24) & 0xff)
#define GITS_TYPER_PTA (1ULL << 19)
#define GITS_TYPER_DEVBITS(x) (((x) >> 13) & 0x1f)
#define GITS_TYPER_ITE_SZ(x) (((x) >> 4) & 0xf)
#define GITS_TYPER_PHYS (1ULL << 0)
#define GITS_CBASER 0x0080
#define GITS_CBASER_VALID (1ULL << 63)
#define GITS_CBASER_IC_NORM_NC (1ULL << 59)
#define GITS_CBASER_MASK 0x1ffffffffff000ULL
#define GITS_CWRITER 0x0088
#define GITS_CREADR 0x0090
#define GITS_BASER(i) (0x0100 + ((i) * 8))
#define GITS_BASER_VALID (1ULL << 63)
#define GITS_BASER_INDIRECT (1ULL << 62)
#define GITS_BASER_IC_NORM_NC (1ULL << 59)
#define GITS_BASER_TYPE_MASK (7ULL << 56)
#define GITS_BASER_TYPE_DEVICE (1ULL << 56)
#define GITS_BASER_TYPE_COLL (4ULL << 56)
#define GITS_BASER_TTE_SZ(x) (((x) >> 48) & 0x1f)
#define GITS_BASER_PGSZ_MASK (3ULL << 8)
#define GITS_BASER_PGSZ_4K (0ULL << 8)
#define GITS_BASER_PGSZ_16K (1ULL << 8)
#define GITS_BASER_PGSZ_64K (2ULL << 8)
#define GITS_BASER_PA_MASK 0x7ffffffff000ULL
#define GITS_TRANSLATER 0x10040
#define GITS_NUM_BASER 8
struct gits_cmd {
uint8_t cmd;
uint32_t deviceid;
uint32_t eventid;
uint32_t intid;
uint64_t dw2;
uint64_t dw3;
};
#define GITS_CMD_VALID (1ULL << 63)
/* ITS commands */
#define SYNC 0x05
#define MAPD 0x08
#define MAPC 0x09
#define MAPTI 0x0a
#define INV 0x0c
#define INVALL 0x0d
#define DISCARD 0x0f
#define GITS_CMDQ_SIZE (64 * 1024)
#define GITS_CMDQ_NENTRIES (GITS_CMDQ_SIZE / sizeof(struct gits_cmd))
struct agintc_msi_device {
LIST_ENTRY(agintc_msi_device) md_list;
uint32_t md_deviceid;
uint32_t md_events;
struct agintc_dmamem *md_itt;
};
int agintc_msi_match(struct device *, void *, void *);
void agintc_msi_attach(struct device *, struct device *, void *);
void *agintc_intr_establish_msi(void *, uint64_t *, uint64_t *,
int , struct cpu_info *, int (*)(void *), void *, char *);
void agintc_intr_disestablish_msi(void *);
void agintc_intr_barrier_msi(void *);
struct agintc_msi_softc {
struct device sc_dev;
bus_space_tag_t sc_iot;
bus_space_handle_t sc_ioh;
bus_dma_tag_t sc_dmat;
bus_addr_t sc_msi_addr;
int sc_msi_delta;
struct agintc_dmamem *sc_cmdq;
uint16_t sc_cmdidx;
int sc_devbits;
struct agintc_dmamem *sc_dtt;
size_t sc_dtt_pgsz;
uint8_t sc_dte_sz;
int sc_cidbits;
struct agintc_dmamem *sc_ctt;
size_t sc_ctt_pgsz;
uint8_t sc_cte_sz;
uint8_t sc_ite_sz;
LIST_HEAD(, agintc_msi_device) sc_msi_devices;
struct interrupt_controller sc_ic;
};
const struct cfattach agintcmsi_ca = {
sizeof (struct agintc_msi_softc), agintc_msi_match, agintc_msi_attach
};
struct cfdriver agintcmsi_cd = {
NULL, "agintcmsi", DV_DULL
};
void agintc_msi_send_cmd(struct agintc_msi_softc *, struct gits_cmd *);
void agintc_msi_wait_cmd(struct agintc_msi_softc *);
int
agintc_msi_match(struct device *parent, void *cfdata, void *aux)
{
struct fdt_attach_args *faa = aux;
return OF_is_compatible(faa->fa_node, "arm,gic-v3-its");
}
void
agintc_msi_attach(struct device *parent, struct device *self, void *aux)
{
struct agintc_msi_softc *sc = (struct agintc_msi_softc *)self;
struct fdt_attach_args *faa = aux;
struct gits_cmd cmd;
uint32_t pre_its[2];
uint64_t typer;
int i, hwcpu;
if (faa->fa_nreg < 1) {
printf(": no registers\n");
return;
}
sc->sc_iot = faa->fa_iot;
if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr,
faa->fa_reg[0].size, 0, &sc->sc_ioh)) {
printf(": can't map registers\n");
return;
}
sc->sc_dmat = faa->fa_dmat;
sc->sc_msi_addr = faa->fa_reg[0].addr + GITS_TRANSLATER;
if (OF_getpropintarray(faa->fa_node, "socionext,synquacer-pre-its",
pre_its, sizeof(pre_its)) == sizeof(pre_its)) {
sc->sc_msi_addr = pre_its[0];
sc->sc_msi_delta = 4;
}
typer = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_TYPER);
if ((typer & GITS_TYPER_PHYS) == 0 || typer & GITS_TYPER_PTA) {
printf(": unsupported type 0x%016llx\n", typer);
goto unmap;
}
sc->sc_ite_sz = GITS_TYPER_ITE_SZ(typer) + 1;
sc->sc_devbits = GITS_TYPER_DEVBITS(typer) + 1;
if (typer & GITS_TYPER_CIL)
sc->sc_cidbits = GITS_TYPER_CIDBITS(typer) + 1;
else
sc->sc_cidbits = 16;
/* Set up command queue. */
sc->sc_cmdq = agintc_dmamem_alloc(sc->sc_dmat,
GITS_CMDQ_SIZE, GITS_CMDQ_SIZE);
if (sc->sc_cmdq == NULL) {
printf(": can't alloc command queue\n");
goto unmap;
}
bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_CBASER,
AGINTC_DMA_DVA(sc->sc_cmdq) | GITS_CBASER_IC_NORM_NC |
(GITS_CMDQ_SIZE / PAGE_SIZE) - 1 | GITS_CBASER_VALID);
/* Set up device translation table. */
for (i = 0; i < GITS_NUM_BASER; i++) {
uint64_t baser;
paddr_t dtt_pa;
size_t size;
baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i));
if ((baser & GITS_BASER_TYPE_MASK) != GITS_BASER_TYPE_DEVICE)
continue;
/* Determine the maximum supported page size. */
bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i),
(baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_64K);
baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i));
if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_64K)
goto dfound;
bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i),
(baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_16K);
baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i));
if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_16K)
goto dfound;
bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i),
(baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_4K);
baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i));
dfound:
switch (baser & GITS_BASER_PGSZ_MASK) {
case GITS_BASER_PGSZ_4K:
sc->sc_dtt_pgsz = PAGE_SIZE;
break;
case GITS_BASER_PGSZ_16K:
sc->sc_dtt_pgsz = 4 * PAGE_SIZE;
break;
case GITS_BASER_PGSZ_64K:
sc->sc_dtt_pgsz = 16 * PAGE_SIZE;
break;
}
/* Calculate table size. */
sc->sc_dte_sz = GITS_BASER_TTE_SZ(baser) + 1;
size = (1ULL << sc->sc_devbits) * sc->sc_dte_sz;
size = roundup(size, sc->sc_dtt_pgsz);
/* Allocate table. */
sc->sc_dtt = agintc_dmamem_alloc(sc->sc_dmat,
size, sc->sc_dtt_pgsz);
if (sc->sc_dtt == NULL) {
printf(": can't alloc translation table\n");
goto unmap;
}
/* Configure table. */
dtt_pa = AGINTC_DMA_DVA(sc->sc_dtt);
KASSERT((dtt_pa & GITS_BASER_PA_MASK) == dtt_pa);
bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i),
GITS_BASER_IC_NORM_NC | baser & GITS_BASER_PGSZ_MASK |
dtt_pa | (size / sc->sc_dtt_pgsz) - 1 | GITS_BASER_VALID);
}
/* Set up collection translation table. */
for (i = 0; i < GITS_NUM_BASER; i++) {
uint64_t baser;
paddr_t ctt_pa;
size_t size;
baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i));
if ((baser & GITS_BASER_TYPE_MASK) != GITS_BASER_TYPE_COLL)
continue;
/* Determine the maximum supported page size. */
bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i),
(baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_64K);
baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i));
if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_64K)
goto cfound;
bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i),
(baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_16K);
baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i));
if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_16K)
goto cfound;
bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i),
(baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_4K);
baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i));
cfound:
switch (baser & GITS_BASER_PGSZ_MASK) {
case GITS_BASER_PGSZ_4K:
sc->sc_ctt_pgsz = PAGE_SIZE;
break;
case GITS_BASER_PGSZ_16K:
sc->sc_ctt_pgsz = 4 * PAGE_SIZE;
break;
case GITS_BASER_PGSZ_64K:
sc->sc_ctt_pgsz = 16 * PAGE_SIZE;
break;
}
/* Calculate table size. */
sc->sc_cte_sz = GITS_BASER_TTE_SZ(baser) + 1;
size = (1ULL << sc->sc_cidbits) * sc->sc_cte_sz;
size = roundup(size, sc->sc_ctt_pgsz);
/* Allocate table. */
sc->sc_ctt = agintc_dmamem_alloc(sc->sc_dmat,
size, sc->sc_ctt_pgsz);
if (sc->sc_ctt == NULL) {
printf(": can't alloc translation table\n");
goto unmap;
}
/* Configure table. */
ctt_pa = AGINTC_DMA_DVA(sc->sc_ctt);
KASSERT((ctt_pa & GITS_BASER_PA_MASK) == ctt_pa);
bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i),
GITS_BASER_IC_NORM_NC | baser & GITS_BASER_PGSZ_MASK |
ctt_pa | (size / sc->sc_ctt_pgsz) - 1 | GITS_BASER_VALID);
}
/* Enable ITS. */
bus_space_write_4(sc->sc_iot, sc->sc_ioh, GITS_CTLR,
GITS_CTLR_ENABLED);
LIST_INIT(&sc->sc_msi_devices);
/* Create one collection per core. */
KASSERT(ncpus <= agintc_sc->sc_num_redist);
for (i = 0; i < ncpus; i++) {
hwcpu = agintc_sc->sc_cpuremap[i];
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = MAPC;
cmd.dw2 = GITS_CMD_VALID |
(agintc_sc->sc_processor[hwcpu] << 16) | i;
agintc_msi_send_cmd(sc, &cmd);
agintc_msi_wait_cmd(sc);
}
printf("\n");
sc->sc_ic.ic_node = faa->fa_node;
sc->sc_ic.ic_cookie = sc;
sc->sc_ic.ic_establish_msi = agintc_intr_establish_msi;
sc->sc_ic.ic_disestablish = agintc_intr_disestablish_msi;
sc->sc_ic.ic_barrier = agintc_intr_barrier_msi;
sc->sc_ic.ic_gic_its_id = OF_getpropint(faa->fa_node,
"openbsd,gic-its-id", 0);
arm_intr_register_fdt(&sc->sc_ic);
return;
unmap:
if (sc->sc_dtt)
agintc_dmamem_free(sc->sc_dmat, sc->sc_dtt);
if (sc->sc_cmdq)
agintc_dmamem_free(sc->sc_dmat, sc->sc_cmdq);
bus_space_unmap(sc->sc_iot, sc->sc_ioh, faa->fa_reg[0].size);
}
void
agintc_msi_send_cmd(struct agintc_msi_softc *sc, struct gits_cmd *cmd)
{
struct gits_cmd *queue = AGINTC_DMA_KVA(sc->sc_cmdq);
memcpy(&queue[sc->sc_cmdidx], cmd, sizeof(*cmd));
/* Make globally visible. */
cpu_dcache_wb_range((vaddr_t)&queue[sc->sc_cmdidx], sizeof(*cmd));
__asm volatile("dsb sy");
sc->sc_cmdidx++;
sc->sc_cmdidx %= GITS_CMDQ_NENTRIES;
bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_CWRITER,
sc->sc_cmdidx * sizeof(*cmd));
}
void
agintc_msi_wait_cmd(struct agintc_msi_softc *sc)
{
uint64_t creadr;
int timo;
for (timo = 1000; timo > 0; timo--) {
creadr = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_CREADR);
if (creadr == sc->sc_cmdidx * sizeof(struct gits_cmd))
break;
delay(1);
}
if (timo == 0)
printf("%s: command queue timeout\n", sc->sc_dev.dv_xname);
}
struct agintc_msi_device *
agintc_msi_create_device(struct agintc_msi_softc *sc, uint32_t deviceid)
{
struct agintc_msi_device *md;
struct gits_cmd cmd;
md = malloc(sizeof(*md), M_DEVBUF, M_ZERO | M_WAITOK);
md->md_deviceid = deviceid;
md->md_itt = agintc_dmamem_alloc(sc->sc_dmat,
32 * sc->sc_ite_sz, PAGE_SIZE);
LIST_INSERT_HEAD(&sc->sc_msi_devices, md, md_list);
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = MAPD;
cmd.deviceid = deviceid;
cmd.eventid = 4; /* size */
cmd.dw2 = AGINTC_DMA_DVA(md->md_itt) | GITS_CMD_VALID;
agintc_msi_send_cmd(sc, &cmd);
agintc_msi_wait_cmd(sc);
return md;
}
struct agintc_msi_device *
agintc_msi_find_device(struct agintc_msi_softc *sc, uint32_t deviceid)
{
struct agintc_msi_device *md;
LIST_FOREACH(md, &sc->sc_msi_devices, md_list) {
if (md->md_deviceid == deviceid)
return md;
}
return agintc_msi_create_device(sc, deviceid);
}
void
agintc_msi_discard(struct agintc_lpi_info *li)
{
struct agintc_msi_softc *sc;
struct cpu_info *ci;
struct gits_cmd cmd;
int hwcpu;
sc = li->li_msic;
ci = li->li_ci;
hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid];
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = DISCARD;
cmd.deviceid = li->li_deviceid;
cmd.eventid = li->li_eventid;
agintc_msi_send_cmd(sc, &cmd);
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = SYNC;
cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16;
agintc_msi_send_cmd(sc, &cmd);
agintc_msi_wait_cmd(sc);
}
void
agintc_msi_inv(struct agintc_lpi_info *li)
{
struct agintc_msi_softc *sc;
struct cpu_info *ci;
struct gits_cmd cmd;
int hwcpu;
sc = li->li_msic;
ci = li->li_ci;
hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid];
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = INV;
cmd.deviceid = li->li_deviceid;
cmd.eventid = li->li_eventid;
agintc_msi_send_cmd(sc, &cmd);
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = SYNC;
cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16;
agintc_msi_send_cmd(sc, &cmd);
agintc_msi_wait_cmd(sc);
}
void *
agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data,
int level, struct cpu_info *ci, int (*func)(void *), void *arg, char *name)
{
struct agintc_msi_softc *sc = (struct agintc_msi_softc *)self;
struct agintc_msi_device *md;
struct gits_cmd cmd;
uint32_t deviceid = *data;
uint32_t eventid;
int i, hwcpu;
if (ci == NULL)
ci = &cpu_info_primary;
hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid];
md = agintc_msi_find_device(sc, deviceid);
if (md == NULL)
return NULL;
eventid = *addr;
if (eventid > 0 && (md->md_events & (1U << eventid)))
return NULL;
for (; eventid < 32; eventid++) {
if ((md->md_events & (1U << eventid)) == 0) {
md->md_events |= (1U << eventid);
break;
}
}
if (eventid >= 32)
return NULL;
for (i = 0; i < agintc_sc->sc_nlpi; i++) {
if (agintc_sc->sc_lpi[i] != NULL)
continue;
agintc_sc->sc_lpi[i] = malloc(sizeof(struct agintc_lpi_info),
M_DEVBUF, M_WAITOK | M_ZERO);
agintc_sc->sc_lpi[i]->li_msic = sc;
agintc_sc->sc_lpi[i]->li_ci = ci;
agintc_sc->sc_lpi[i]->li_deviceid = deviceid;
agintc_sc->sc_lpi[i]->li_eventid = eventid;
agintc_sc->sc_lpi[i]->li_ih =
agintc_intr_establish(LPI_BASE + i,
IST_EDGE_RISING, level, ci, func, arg, name);
if (agintc_sc->sc_lpi[i]->li_ih == NULL) {
free(agintc_sc->sc_lpi[i], M_DEVBUF,
sizeof(struct agintc_lpi_info));
agintc_sc->sc_lpi[i] = NULL;
return NULL;
}
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = MAPTI;
cmd.deviceid = deviceid;
cmd.eventid = eventid;
cmd.intid = LPI_BASE + i;
cmd.dw2 = ci->ci_cpuid;
agintc_msi_send_cmd(sc, &cmd);
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = SYNC;
cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16;
agintc_msi_send_cmd(sc, &cmd);
agintc_msi_wait_cmd(sc);
*addr = sc->sc_msi_addr + deviceid * sc->sc_msi_delta;
*data = eventid;
return &agintc_sc->sc_lpi[i];
}
return NULL;
}
void
agintc_intr_disestablish_msi(void *cookie)
{
struct agintc_lpi_info *li = *(void **)cookie;
agintc_intr_disestablish(li->li_ih);
agintc_msi_discard(li);
agintc_msi_inv(li);
free(li, M_DEVBUF, sizeof(*li));
*(void **)cookie = NULL;
}
void
agintc_intr_barrier_msi(void *cookie)
{
struct agintc_lpi_info *li = *(void **)cookie;
agintc_intr_barrier(li->li_ih);
}
struct agintc_dmamem *
agintc_dmamem_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align)
{
struct agintc_dmamem *adm;
int nsegs;
adm = malloc(sizeof(*adm), M_DEVBUF, M_WAITOK | M_ZERO);
adm->adm_size = size;
if (bus_dmamap_create(dmat, size, 1, size, 0,
BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0)
goto admfree;
if (bus_dmamem_alloc(dmat, size, align, 0, &adm->adm_seg, 1,
&nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0)
goto destroy;
if (bus_dmamem_map(dmat, &adm->adm_seg, nsegs, size,
&adm->adm_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE) != 0)
goto free;
if (bus_dmamap_load_raw(dmat, adm->adm_map, &adm->adm_seg,
nsegs, size, BUS_DMA_WAITOK) != 0)
goto unmap;
/* Make globally visible. */
cpu_dcache_wb_range((vaddr_t)adm->adm_kva, size);
__asm volatile("dsb sy");
return adm;
unmap:
bus_dmamem_unmap(dmat, adm->adm_kva, size);
free:
bus_dmamem_free(dmat, &adm->adm_seg, 1);
destroy:
bus_dmamap_destroy(dmat, adm->adm_map);
admfree:
free(adm, M_DEVBUF, sizeof(*adm));
return NULL;
}
void
agintc_dmamem_free(bus_dma_tag_t dmat, struct agintc_dmamem *adm)
{
bus_dmamem_unmap(dmat, adm->adm_kva, adm->adm_size);
bus_dmamem_free(dmat, &adm->adm_seg, 1);
bus_dmamap_destroy(dmat, adm->adm_map);
free(adm, M_DEVBUF, sizeof(*adm));
}
Reference in a new issue View git blame Copy permalink