From 65158cc91a6a7ecbee1905a3a405ef72165e8b03 Mon Sep 17 00:00:00 2001 From: GuEe-GUI <2991707448@qq.com> Date: Mon, 27 May 2024 10:14:37 +0800 Subject: [PATCH] PIC: Add ARM GICv2/v3, and V2M, ITS support. GIC (Generic Interrupt Controller) is the ARMv7/v8 PIC. V2M is the PCI MSI/MSI-X for GICv2. ITS is the PCI MSI/MSI-X for GICv3/v4. Signed-off-by: GuEe-GUI <2991707448@qq.com> --- components/drivers/include/dt-bindings/size.h | 17 + components/drivers/pic/Kconfig | 32 + components/drivers/pic/SConscript | 15 + components/drivers/pic/pic-gic-common.c | 183 ++ components/drivers/pic/pic-gic-common.h | 59 + components/drivers/pic/pic-gicv2.c | 659 +++++++ components/drivers/pic/pic-gicv2.h | 83 + components/drivers/pic/pic-gicv2m.c | 378 ++++ components/drivers/pic/pic-gicv3-its.c | 1529 +++++++++++++++++ components/drivers/pic/pic-gicv3.c | 1096 ++++++++++++ components/drivers/pic/pic-gicv3.h | 390 +++++ 11 files changed, 4441 insertions(+) create mode 100644 components/drivers/include/dt-bindings/size.h create mode 100644 components/drivers/pic/pic-gic-common.c create mode 100644 components/drivers/pic/pic-gic-common.h create mode 100644 components/drivers/pic/pic-gicv2.c create mode 100644 components/drivers/pic/pic-gicv2.h create mode 100755 components/drivers/pic/pic-gicv2m.c create mode 100755 components/drivers/pic/pic-gicv3-its.c create mode 100644 components/drivers/pic/pic-gicv3.c create mode 100644 components/drivers/pic/pic-gicv3.h diff --git a/components/drivers/include/dt-bindings/size.h b/components/drivers/include/dt-bindings/size.h new file mode 100644 index 000000000000..8fe7c1e43982 --- /dev/null +++ b/components/drivers/include/dt-bindings/size.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __DT_BINDINGS_SIZE_H__ +#define __DT_BINDINGS_SIZE_H__ + +#define SIZE_KB 1024 +#define SIZE_MB (1024 * SIZE_KB) +#define SIZE_GB (1024 * SIZE_MB) + +#define SIZE_ALIGN(size, align) (((size) + (align) - 1) & ~((align) - 1)) +#define SIZE_ALIGN_DOWN(size, align) ((size) & ~((align) - 1)) + +#endif /* __DT_BINDINGS_SIZE_H__ */ diff --git a/components/drivers/pic/Kconfig b/components/drivers/pic/Kconfig index 16c00ddce3e2..346ab3a614d8 100644 --- a/components/drivers/pic/Kconfig +++ b/components/drivers/pic/Kconfig @@ -9,3 +9,35 @@ config MAX_HANDLERS depends on RT_USING_PIC range 1 2147483647 default 256 + +config RT_PIC_ARM_GIC + bool "ARM GICv2/v1" + depends on RT_USING_PIC + depends on RT_USING_OFW + default n + +config RT_PIC_ARM_GIC_V2M + bool "ARM GIC V2M" if RT_PIC_ARM_GIC && RT_PCI_MSI + depends on RT_USING_OFW + select RT_USING_ADT_BITMAP + default n + +config RT_PIC_ARM_GIC_V3 + bool "ARM GICv3" + depends on RT_USING_PIC + depends on RT_USING_OFW + default n + +config RT_PIC_ARM_GIC_V3_ITS + bool "ARM GICv3 ITS (Interrupt Translation Service)" if RT_PIC_ARM_GIC_V3 && RT_PCI_MSI + depends on RT_USING_OFW + select RT_USING_ADT_REF + select RT_USING_ADT_BITMAP + default n + +config RT_PIC_ARM_GIC_MAX_NR + int + depends on RT_USING_PIC + depends on RT_PIC_ARM_GIC + default 2 if SOC_REALVIEW + default 1 diff --git a/components/drivers/pic/SConscript b/components/drivers/pic/SConscript index ade23fc3098a..3f67c53e2016 100644 --- a/components/drivers/pic/SConscript +++ b/components/drivers/pic/SConscript @@ -10,6 +10,21 @@ CPPPATH = [cwd + '/../include'] src = ['pic.c'] +if GetDepend(['RT_PIC_ARM_GIC']) or GetDepend(['RT_PIC_ARM_GIC_V3']): + src += ['pic-gic-common.c'] + +if GetDepend(['RT_PIC_ARM_GIC']): + src += ['pic-gicv2.c'] + +if GetDepend(['RT_PIC_ARM_GIC_V2M']): + src += ['pic-gicv2m.c'] + +if GetDepend(['RT_PIC_ARM_GIC_V3']): + src += ['pic-gicv3.c'] + +if GetDepend(['RT_PIC_ARM_GIC_V3_ITS']): + src += ['pic-gicv3-its.c'] + group = DefineGroup('DeviceDrivers', src, depend = [''], CPPPATH = CPPPATH) Return('group') diff --git a/components/drivers/pic/pic-gic-common.c b/components/drivers/pic/pic-gic-common.c new file mode 100644 index 000000000000..3c7379444965 --- /dev/null +++ b/components/drivers/pic/pic-gic-common.c @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#include +#include + +#define DBG_TAG "pic.gic*" +#define DBG_LVL DBG_INFO +#include + +#include + +#include "pic-gicv2.h" +#include "pic-gic-common.h" + +void gic_common_init_quirk_ofw(const struct rt_ofw_node *ic_np, const struct gic_quirk *quirks, void *data) +{ + for (; quirks->desc; ++quirks) + { + if (!quirks->compatible || !rt_ofw_node_is_compatible(ic_np, quirks->compatible)) + { + continue; + } + + RT_ASSERT(quirks->init != RT_NULL); + + if (!quirks->init(data)) + { + LOG_I("Enable workaround for %s", quirks->desc); + } + } +} + +void gic_common_init_quirk_hw(rt_uint32_t iidr, const struct gic_quirk *quirks, void *data) +{ + for (; quirks->desc; ++quirks) + { + if (quirks->compatible) + { + continue; + } + + if (quirks->iidr == (iidr & quirks->iidr_mask)) + { + RT_ASSERT(quirks->init != RT_NULL); + + if (!quirks->init(data)) + { + LOG_I("Enable workaround for %s", quirks->desc); + } + } + } +} + +void gic_common_sgi_config(void *base, void *data, int irq_base) +{ +#ifdef RT_USING_SMP + if (irq_base < 2) + { + struct rt_pic_irq *pirq; + +#define DECLARE_GIC_IPI(ipi, hwirq) \ + rt_pic_config_ipi(data, ipi, hwirq); \ + pirq = rt_pic_find_ipi(data, ipi); \ + pirq->mode = RT_IRQ_MODE_EDGE_RISING; \ + + DECLARE_GIC_IPI(RT_SCHEDULE_IPI, 0); + DECLARE_GIC_IPI(RT_STOP_IPI, 1); + +#undef DECLARE_GIC_IPI + } +#endif /* RT_USING_SMP */ +} + +rt_err_t gic_common_configure_irq(void *base, int irq, rt_uint32_t mode, void (*sync_access)(void *), void *data) +{ + rt_err_t err = RT_EOK; + rt_ubase_t level; + rt_uint32_t val, oldval; + rt_uint32_t confoff = (irq / 16) * 4; + rt_uint32_t confmask = 0x2 << ((irq % 16) * 2); + static struct rt_spinlock ic_lock = { 0 }; + + level = rt_spin_lock_irqsave(&ic_lock); + + val = oldval = HWREG32(base + confoff); + + if (mode & RT_IRQ_MODE_LEVEL_MASK) + { + /* Level-sensitive */ + val &= ~confmask; + } + else if (mode & RT_IRQ_MODE_EDGE_BOTH) + { + /* Edge-triggered */ + val |= confmask; + } + + if (val != oldval) + { + HWREG32(base + confoff) = val; + + if (HWREG32(base + confoff) != val) + { + err = -RT_EINVAL; + } + if (sync_access) + { + sync_access(data); + } + } + + rt_spin_unlock_irqrestore(&ic_lock, level); + + return err; +} + +void gic_common_dist_config(void *base, int max_irqs, void (*sync_access)(void *), void *data) +{ + rt_uint32_t i; + + /* Set all global interrupts to be level triggered, active low. */ + for (i = 32; i < max_irqs; i += 16) + { + HWREG32(base + GIC_DIST_CONFIG + i / 4) = GICD_INT_ACTLOW_LVLTRIG; + } + + /* Set priority on all global interrupts. */ + for (i = 32; i < max_irqs; i += 4) + { + HWREG32(base + GIC_DIST_PRI + i * 4 / 4) = GICD_INT_DEF_PRI_X4; + } + + /* Disable all SPIs. */ + for (i = 32; i < max_irqs; i += 32) + { + HWREG32(base + GIC_DIST_ACTIVE_CLEAR + i / 8) = GICD_INT_EN_CLR_X32; + HWREG32(base + GIC_DIST_ENABLE_CLEAR + i / 8) = GICD_INT_EN_CLR_X32; + } + + if (sync_access) + { + sync_access(data); + } +} + +void gic_common_cpu_config(void *base, int nr, void (*sync_access)(void *), void *data) +{ + rt_uint32_t i; + + /* Disable all SGIs, PPIs. */ + for (i = 0; i < nr; i += 32) + { + HWREG32(base + GIC_DIST_ACTIVE_CLEAR + i / 8) = GICD_INT_EN_CLR_X32; + HWREG32(base + GIC_DIST_ENABLE_CLEAR + i / 8) = GICD_INT_EN_CLR_X32; + } + + /* Set priority on all PPI and SGI. */ + for (i = 0; i < nr; i += 4) + { + HWREG32(base + GIC_DIST_PRI + i * 4 / 4) = GICD_INT_DEF_PRI_X4; + } + + if (sync_access) + { + sync_access(data); + } +} + +void gic_fill_ppi_affinity(bitmap_t *affinity) +{ + for (int cpuid = 0; cpuid < RT_CPUS_NR; ++cpuid) + { + RT_IRQ_AFFINITY_SET(affinity, cpuid); + } +} diff --git a/components/drivers/pic/pic-gic-common.h b/components/drivers/pic/pic-gic-common.h new file mode 100644 index 000000000000..62cf5ea45103 --- /dev/null +++ b/components/drivers/pic/pic-gic-common.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#ifndef __IRQ_GIC_COMMON_H__ +#define __IRQ_GIC_COMMON_H__ + +#include + +#ifdef RT_PCI_MSI +#include +#endif +#include + +#define GIC_SGI_NR 16 + +#define GICD_INT_DEF_PRI 0xa0U +#define GICD_INT_DEF_PRI_X4 \ +( \ + (GICD_INT_DEF_PRI << 24) | \ + (GICD_INT_DEF_PRI << 16) | \ + (GICD_INT_DEF_PRI << 8) | \ + GICD_INT_DEF_PRI \ +) + +struct gic_quirk +{ + const char *desc; + const char *compatible; + rt_err_t (*init)(void *data); + + rt_uint32_t iidr; + rt_uint32_t iidr_mask; +}; + +void gic_common_init_quirk_ofw(const struct rt_ofw_node *ic_np, const struct gic_quirk *quirks, void *data); +void gic_common_init_quirk_hw(rt_uint32_t iidr, const struct gic_quirk *quirks, void *data); + +void gic_common_sgi_config(void *base, void *data, int irq_base); +rt_err_t gic_common_configure_irq(void *base, int irq, rt_uint32_t mode, void (*sync_access)(void *), void *data); +void gic_common_dist_config(void *base, int max_irqs, void (*sync_access)(void *), void *data); +void gic_common_cpu_config(void *base, int nr, void (*sync_access)(void *), void *data); + +void gic_fill_ppi_affinity(bitmap_t *affinity); + +#ifdef RT_PIC_ARM_GIC_V2M +rt_err_t gicv2m_ofw_probe(struct rt_ofw_node *ic_np, const struct rt_ofw_node_id *id); +#endif +#ifdef RT_PIC_ARM_GIC_V3_ITS +rt_err_t gicv3_its_ofw_probe(struct rt_ofw_node *ic_np, const struct rt_ofw_node_id *id); +#endif + +#endif /* __IRQ_GIC_COMMON_H__ */ diff --git a/components/drivers/pic/pic-gicv2.c b/components/drivers/pic/pic-gicv2.c new file mode 100644 index 000000000000..422ab487e3dd --- /dev/null +++ b/components/drivers/pic/pic-gicv2.c @@ -0,0 +1,659 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2013-07-20 Bernard first version + * 2014-04-03 Grissiom many enhancements + * 2018-11-22 Jesven add rt_hw_ipi_send() + * add rt_hw_ipi_handler_install() + * 2022-08-24 GuEe-GUI add pic support + * 2022-11-07 GuEe-GUI add v2m support + */ + +#include +#include +#include + +#define DBG_TAG "pic.gicv2" +#define DBG_LVL DBG_INFO +#include + +#include + +#include + +#include "pic-gicv2.h" +#include "pic-gic-common.h" + +#define GIC_CPU_IMAX 8 + +#define raw_to_gicv2(raw) rt_container_of(raw, struct gicv2, parent) + +static rt_bool_t needs_rmw_access = RT_FALSE; +static int _gicv2_nr = 0, _init_cpu_id = 0; +static struct gicv2 _gicv2_list[RT_PIC_ARM_GIC_MAX_NR] = {}; +static rt_bool_t _gicv2_eoi_mode_ns = RT_FALSE; +static rt_uint8_t _gicv2_cpumask_map[GIC_CPU_IMAX] = +{ + [0 ... GIC_CPU_IMAX - 1] = 0xff, +}; + +static rt_uint8_t gicv2_cpumask_map(struct gicv2 *gic) +{ + rt_uint32_t mask, i; + + for (i = mask = 0; i < 32; i += 4) + { + mask = HWREG32(gic->dist_base + GIC_DIST_TARGET + i); + mask |= mask >> 16; + mask |= mask >> 8; + + if (mask) + { + break; + } + } + + return mask; +} + +static void gicv2_dist_init(struct gicv2 *gic) +{ + void *base = gic->dist_base; + rt_uint32_t i; + rt_uint32_t cpumask = gicv2_cpumask_map(gic); + + _init_cpu_id = rt_hw_cpu_id(); + + gic->max_irq = HWREG32(base + GIC_DIST_TYPE) & 0x1f; + gic->max_irq = (gic->max_irq + 1) * 32; + + /* + * The GIC only supports up to 1020 interrupt sources. + * Limit this to either the architected maximum, or the + * platform maximum. + */ + if (gic->max_irq > 1020) + { + gic->max_irq = 1020; + } + + LOG_D("Max irq = %d", gic->max_irq); + + HWREG32(base + GIC_DIST_CTRL) = GICD_DISABLE; + + /* Set all global (unused) interrupts to this CPU only. */ + cpumask |= cpumask << 8; + cpumask |= cpumask << 16; + + for (i = 32; i < gic->max_irq; i += 4) + { + HWREG32(base + GIC_DIST_TARGET + i * 4 / 4) = cpumask; + } + + gic_common_dist_config(base, gic->max_irq, RT_NULL, RT_NULL); + + HWREG32(base + GIC_DIST_CTRL) = GICD_ENABLE; +} + +static void gicv2_cpu_init(struct gicv2 *gic) +{ + rt_uint32_t cpumask; + void *base = gic->cpu_base; + rt_uint32_t config = GICC_ENABLE; + int cpu_id = rt_hw_cpu_id(); + + cpumask = gicv2_cpumask_map(gic); + _gicv2_cpumask_map[cpu_id] = cpumask; + + /* + * Clear our mask from the other map entries in case they're + * still undefined. + */ + for (int i = 0; i < RT_ARRAY_SIZE(_gicv2_cpumask_map); ++i) + { + if (i != cpu_id) + { + _gicv2_cpumask_map[i] &= ~cpumask; + } + } + + gic_common_cpu_config(gic->dist_base, 32, RT_NULL, RT_NULL); + + HWREG32(base + GIC_CPU_PRIMASK) = GICC_INT_PRI_THRESHOLD; + HWREG32(base + GIC_CPU_BINPOINT) = 0x7; + +#ifdef ARCH_SUPPORT_HYP + _gicv2_eoi_mode_ns = RT_TRUE; +#else + _gicv2_eoi_mode_ns = !!rt_ofw_bootargs_select("pic.gicv2_eoimode", 0); +#endif + + if (_gicv2_eoi_mode_ns) + { + config |= GIC_CPU_CTRL_EOI_MODE_NS; + } + + HWREG32(base + GIC_CPU_CTRL) = config; +} + +static rt_err_t gicv2_irq_init(struct rt_pic *pic) +{ + gicv2_cpu_init(rt_container_of(pic, struct gicv2, parent)); + + return RT_EOK; +} + +static void gicv2_irq_ack(struct rt_pic_irq *pirq) +{ + int hwirq = pirq->hwirq; + struct gicv2 *gic = raw_to_gicv2(pirq->pic); + + if (!_gicv2_eoi_mode_ns) + { + HWREG32(gic->dist_base + GIC_DIST_PENDING_CLEAR + hwirq / 32 * 4) = 1U << (hwirq % 32); + } + + HWREG32(gic->cpu_base + GIC_CPU_EOI) = hwirq; +} + +static void gicv2_irq_mask(struct rt_pic_irq *pirq) +{ + int hwirq = pirq->hwirq; + struct gicv2 *gic = raw_to_gicv2(pirq->pic); + + HWREG32(gic->dist_base + GIC_DIST_ENABLE_CLEAR + hwirq / 32 * 4) = 1U << (hwirq % 32); +} + +static void gicv2_irq_unmask(struct rt_pic_irq *pirq) +{ + int hwirq = pirq->hwirq; + struct gicv2 *gic = raw_to_gicv2(pirq->pic); + + HWREG32(gic->dist_base + GIC_DIST_ENABLE_SET + hwirq / 32 * 4) = 1U << (hwirq % 32); +} + +static void gicv2_irq_eoi(struct rt_pic_irq *pirq) +{ + struct gicv2 *gic = raw_to_gicv2(pirq->pic); + + if (_gicv2_eoi_mode_ns) + { + HWREG32(gic->cpu_base + GIC_CPU_DIR) = pirq->hwirq; + } +} + +static rt_err_t gicv2_irq_set_priority(struct rt_pic_irq *pirq, rt_uint32_t priority) +{ + rt_uint32_t mask; + int hwirq = pirq->hwirq; + struct gicv2 *gic = raw_to_gicv2(pirq->pic); + + mask = HWREG32(gic->dist_base + GIC_DIST_PRI + hwirq / 4 * 4); + mask &= ~(0xffU << ((hwirq % 4) * 8)); + mask |= ((priority & 0xffU) << ((hwirq % 4) * 8)); + HWREG32(gic->dist_base + GIC_DIST_PRI + hwirq / 4 * 4) = mask; + + return RT_EOK; +} + +static rt_err_t gicv2_irq_set_affinity(struct rt_pic_irq *pirq, bitmap_t *affinity) +{ + int hwirq = pirq->hwirq; + struct gicv2 *gic = raw_to_gicv2(pirq->pic); + rt_uint32_t target_list = ((rt_uint8_t *)affinity)[gic - &_gicv2_list[0]]; + rt_uint8_t valb = _gicv2_cpumask_map[__rt_ffs(target_list) - 1]; + void *io_addr = gic->dist_base + GIC_DIST_TARGET + hwirq; + + if (rt_unlikely(valb == 0xfe)) + { + return -RT_EIO; + } + + if (rt_unlikely(needs_rmw_access)) + { + /* RMW write byte */ + rt_uint32_t val; + rt_ubase_t level; + rt_ubase_t offset = (rt_ubase_t)io_addr & 3UL, shift = offset * 8; + static struct rt_spinlock rmw_lock = {}; + + level = rt_spin_lock_irqsave(&rmw_lock); + + io_addr -= offset; + val = HWREG32(io_addr); + val &= ~RT_GENMASK(shift + 7, shift); + val |= valb << shift; + HWREG32(io_addr) = val; + + rt_spin_unlock_irqrestore(&rmw_lock, level); + } + else + { + HWREG8(io_addr) = valb; + } + + return RT_EOK; +} + +static rt_err_t gicv2_irq_set_triger_mode(struct rt_pic_irq *pirq, rt_uint32_t mode) +{ + rt_err_t err = RT_EOK; + int hwirq = pirq->hwirq; + struct gicv2 *gic = raw_to_gicv2(pirq->pic); + + if (hwirq >= GIC_SGI_NR) + { + err = gic_common_configure_irq(gic->dist_base + GIC_DIST_CONFIG, pirq->hwirq, mode, RT_NULL, RT_NULL); + } + else + { + err = -RT_ENOSYS; + } + + return err; +} + +static void gicv2_irq_send_ipi(struct rt_pic_irq *pirq, bitmap_t *cpumask) +{ + struct gicv2 *gic; + int sgi = pirq->hwirq; + rt_uint8_t *target_list = (rt_uint8_t *)cpumask; + + for (int i = 0; i < _gicv2_nr; ++i) + { + if (*target_list) + { + gic = &_gicv2_list[i]; + + HWREG32(gic->dist_base + GIC_DIST_SOFTINT) = ((*target_list & 0xffU) << 16) | (sgi & 0xf); + + rt_hw_dsb(); + } + + ++target_list; + } +} + +static rt_err_t gicv2_irq_set_state(struct rt_pic *pic, int hwirq, int type, rt_bool_t state) +{ + rt_err_t err = RT_EOK; + rt_uint32_t offset = 0; + struct gicv2 *gic = raw_to_gicv2(pic); + + switch (type) + { + case RT_IRQ_STATE_PENDING: + offset = state ? GIC_DIST_PENDING_SET : GIC_DIST_PENDING_CLEAR; + break; + case RT_IRQ_STATE_ACTIVE: + offset = state ? GIC_DIST_ACTIVE_SET : GIC_DIST_ACTIVE_CLEAR; + break; + case RT_IRQ_STATE_MASKED: + offset = state ? GIC_DIST_ENABLE_CLEAR : GIC_DIST_ENABLE_SET; + break; + default: + err = -RT_EINVAL; + break; + } + + if (!err) + { + rt_uint32_t mask = 1 << (hwirq % 32); + + HWREG32(gic->dist_base + offset + (hwirq / 32) * 4) = mask; + } + + return err; +} + +static rt_err_t gicv2_irq_get_state(struct rt_pic *pic, int hwirq, int type, rt_bool_t *out_state) +{ + rt_err_t err = RT_EOK; + rt_uint32_t offset = 0; + struct gicv2 *gic = raw_to_gicv2(pic); + + switch (type) + { + case RT_IRQ_STATE_PENDING: + offset = GIC_DIST_PENDING_SET; + break; + case RT_IRQ_STATE_ACTIVE: + offset = GIC_DIST_ACTIVE_SET; + break; + case RT_IRQ_STATE_MASKED: + offset = GIC_DIST_ENABLE_SET; + break; + default: + err = -RT_EINVAL; + break; + } + + if (!err) + { + rt_uint32_t mask = 1 << (hwirq % 32); + + *out_state = !!(HWREG32(gic->dist_base + offset + (hwirq / 32) * 4) & mask); + } + + return err; +} + +static int gicv2_irq_map(struct rt_pic *pic, int hwirq, rt_uint32_t mode) +{ + int irq, irq_index = hwirq - GIC_SGI_NR; + struct rt_pic_irq *pirq = rt_pic_find_irq(pic, irq_index); + + if (pirq && hwirq >= GIC_SGI_NR) + { + pirq->mode = mode; + pirq->priority = GICD_INT_DEF_PRI; + + if (hwirq < 32) + { + gic_fill_ppi_affinity(pirq->affinity); + } + else + { + RT_IRQ_AFFINITY_SET(pirq->affinity, _init_cpu_id); + } + + irq = rt_pic_config_irq(pic, irq_index, hwirq); + + if (irq >= 0 && mode != RT_IRQ_MODE_LEVEL_HIGH) + { + gicv2_irq_set_triger_mode(pirq, mode); + } + } + else + { + irq = -1; + } + + return irq; +} + +static rt_err_t gicv2_irq_parse(struct rt_pic *pic, struct rt_ofw_cell_args *args, struct rt_pic_irq *out_pirq) +{ + rt_err_t err = RT_EOK; + + if (args->args_count == 3) + { + out_pirq->mode = args->args[2] & RT_IRQ_MODE_MASK; + + switch (args->args[0]) + { + case 0: + /* SPI */ + out_pirq->hwirq = args->args[1] + 32; + break; + case 1: + /* PPI */ + out_pirq->hwirq = args->args[1] + 16; + break; + default: + err = -RT_ENOSYS; + break; + } + } + else + { + err = -RT_EINVAL; + } + + return err; +} + +const static struct rt_pic_ops gicv2_ops = +{ + .name = "GICv2", + .irq_init = gicv2_irq_init, + .irq_ack = gicv2_irq_ack, + .irq_mask = gicv2_irq_mask, + .irq_unmask = gicv2_irq_unmask, + .irq_eoi = gicv2_irq_eoi, + .irq_set_priority = gicv2_irq_set_priority, + .irq_set_affinity = gicv2_irq_set_affinity, + .irq_set_triger_mode = gicv2_irq_set_triger_mode, + .irq_send_ipi = gicv2_irq_send_ipi, + .irq_set_state = gicv2_irq_set_state, + .irq_get_state = gicv2_irq_get_state, + .irq_map = gicv2_irq_map, + .irq_parse = gicv2_irq_parse, +}; + +static rt_bool_t gicv2_handler(void *data) +{ + rt_bool_t res = RT_FALSE; + int hwirq; + struct gicv2 *gic = data; + + hwirq = HWREG32(gic->cpu_base + GIC_CPU_INTACK) & 0x3ffUL; + + if (!(hwirq >= 1020 && hwirq <= 1023)) + { + struct rt_pic_irq *pirq; + + if (hwirq < GIC_SGI_NR) + { + rt_hw_rmb(); + + pirq = rt_pic_find_ipi(&gic->parent, hwirq); + } + else + { + pirq = rt_pic_find_irq(&gic->parent, hwirq - GIC_SGI_NR); + } + + gicv2_irq_ack(pirq); + + rt_pic_handle_isr(pirq); + + gicv2_irq_eoi(pirq); + + res = RT_TRUE; + } + + return res; +} + +static rt_err_t gicv2_enable_rmw_access(void *data) +{ + if (rt_ofw_machine_is_compatible("renesas,emev2")) + { + needs_rmw_access = RT_TRUE; + return RT_EOK; + } + + return -RT_EINVAL; +} + +static const struct gic_quirk _gicv2_quirks[] = +{ + { + .desc = "GICv2: Broken byte access", + .compatible = "arm,pl390", + .init = gicv2_enable_rmw_access, + }, + { /* sentinel */ } +}; + +static rt_err_t gicv2_iomap_init(struct gicv2 *gic, rt_uint64_t *regs) +{ + rt_err_t err = RT_EOK; + int idx; + const char *name[] = + { + "Distributor", + "CPU interfaces", + "Virtual interface control", + "Virtual CPU interface", + }; + + do { + /* GICD->GICC->GICH->GICV */ + gic->dist_size = regs[1]; + gic->dist_base = rt_ioremap((void *)regs[0], gic->dist_size); + if (!gic->dist_base) + { + idx = 0; + err = -RT_ERROR; + break; + } + + gic->cpu_size = regs[3]; + gic->cpu_base = rt_ioremap((void *)regs[2], gic->cpu_size); + if (!gic->cpu_base) + { + idx = 1; + err = -RT_ERROR; + break; + } + + /* ArchRev[4:7] */ + gic->version = HWREG32(gic->dist_base + GIC_DIST_ICPIDR2) >> 4; + + #ifdef ARCH_SUPPORT_HYP + if (gic->version == 1) + { + break; + } + + gic->hyp_size = regs[5]; + gic->hyp_base = rt_ioremap((void *)regs[4], gic->hyp_size); + if (!gic->hyp_base) + { + idx = 2; + err = -RT_ERROR; + break; + } + + gic->vcpu_size = regs[7]; + gic->vcpu_base = rt_ioremap((void *)regs[6], gic->vcpu_size); + if (!gic->vcpu_base) + { + idx = 3; + err = -RT_ERROR; + break; + } + #endif /* ARCH_SUPPORT_HYP */ + } while (0); + + if (err) + { + RT_UNUSED(idx); + RT_UNUSED(name); + + LOG_E("gic[%d] %s IO[%p, %p] map fail", _gicv2_nr, name[idx], regs[idx * 2], regs[idx * 2 + 1]); + } + + return err; +} + +static void gicv2_init(struct gicv2 *gic) +{ + gicv2_dist_init(gic); + + gic->parent.priv_data = gic; + gic->parent.ops = &gicv2_ops; + + rt_pic_linear_irq(&gic->parent, gic->max_irq + 1 - GIC_SGI_NR); + gic_common_sgi_config(gic->dist_base, &gic->parent, _gicv2_nr * GIC_SGI_NR); + + rt_pic_add_traps(gicv2_handler, gic); + + rt_pic_user_extends(&gic->parent); +} + +static void gicv2_init_fail(struct gicv2 *gic) +{ + if (gic->dist_base) + { + rt_iounmap(gic->dist_base); + } + if (gic->cpu_base) + { + rt_iounmap(gic->cpu_base); + } + if (gic->hyp_base) + { + rt_iounmap(gic->hyp_base); + } + if (gic->vcpu_base) + { + rt_iounmap(gic->vcpu_base); + } + rt_memset(gic, 0, sizeof(*gic)); +} + +static rt_err_t gicv2_ofw_init(struct rt_ofw_node *np, const struct rt_ofw_node_id *id) +{ + rt_err_t err = RT_EOK; + struct gicv2 *gic = RT_NULL; + + do { + rt_uint64_t regs[8]; + + if (_gicv2_nr >= RT_PIC_ARM_GIC_MAX_NR) + { + LOG_W("GICv2/v1 table is full"); + err = -RT_EFULL; + break; + } + + gic = &_gicv2_list[_gicv2_nr]; + + rt_ofw_get_address_array(np, RT_ARRAY_SIZE(regs), regs); + + if ((err = gicv2_iomap_init(gic, regs))) + { + break; + } + + if (gic->version != 1 && gic->version != 2) + { + LOG_E("Version = %d is not support", gic->version); + err = -RT_EINVAL; + break; + } + + gic_common_init_quirk_ofw(np, _gicv2_quirks, gic); + gicv2_init(gic); + + rt_ofw_data(np) = &gic->parent; + + if (gic->version == 2) + { + #ifdef RT_PIC_ARM_GIC_V2M + gicv2m_ofw_probe(np, id); + #endif + } + + ++_gicv2_nr; + } while (0); + + if (err && gic) + { + gicv2_init_fail(gic); + } + + return err; +} + +static const struct rt_ofw_node_id gicv2_ofw_ids[] = +{ + { .compatible = "arm,gic-400" }, + { .compatible = "arm,arm11mp-gic" }, + { .compatible = "arm,arm1176jzf-devchip-gic" }, + { .compatible = "arm,cortex-a15-gic" }, + { .compatible = "arm,cortex-a9-gic" }, + { .compatible = "arm,cortex-a7-gic" }, + { .compatible = "qcom,msm-8660-qgic" }, + { .compatible = "qcom,msm-qgic2" }, + { .compatible = "arm,pl390" }, + { /* sentinel */ } +}; +RT_PIC_OFW_DECLARE(gicv2, gicv2_ofw_ids, gicv2_ofw_init); diff --git a/components/drivers/pic/pic-gicv2.h b/components/drivers/pic/pic-gicv2.h new file mode 100644 index 000000000000..69f9651d1f27 --- /dev/null +++ b/components/drivers/pic/pic-gicv2.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2013-07-20 Bernard first version + * 2023-02-01 GuEe-GUI move macros to header + */ + +#ifndef __PIC_GICV2_H__ +#define __PIC_GICV2_H__ + +#include + +#include + +#define GIC_DIST_CTRL 0x000 +#define GIC_DIST_TYPE 0x004 +#define GIC_DIST_IIDR 0x008 +#define GIC_DIST_IGROUP 0x080 +#define GIC_DIST_ENABLE_SET 0x100 +#define GIC_DIST_ENABLE_CLEAR 0x180 +#define GIC_DIST_PENDING_SET 0x200 +#define GIC_DIST_PENDING_CLEAR 0x280 +#define GIC_DIST_ACTIVE_SET 0x300 +#define GIC_DIST_ACTIVE_CLEAR 0x380 +#define GIC_DIST_PRI 0x400 +#define GIC_DIST_TARGET 0x800 +#define GIC_DIST_CONFIG 0xc00 +#define GIC_DIST_SOFTINT 0xf00 +#define GIC_DIST_SGI_PENDING_CLEAR 0xf10 +#define GIC_DIST_SGI_PENDING_SET 0xf20 +#define GIC_DIST_ICPIDR2 0xfe8 + +#define GICD_ENABLE 0x1 +#define GICD_DISABLE 0x0 +#define GICD_INT_ACTLOW_LVLTRIG 0x0 +#define GICD_INT_EN_CLR_X32 0xffffffff +#define GICD_INT_EN_SET_SGI 0x0000ffff +#define GICD_INT_EN_CLR_PPI 0xffff0000 + +#define GICD_GROUP0 0 +#define GICD_GROUP1 (~GICD_GROUP0) + +#define GIC_CPU_CTRL 0x00 +#define GIC_CPU_PRIMASK 0x04 +#define GIC_CPU_BINPOINT 0x08 +#define GIC_CPU_INTACK 0x0c +#define GIC_CPU_EOI 0x10 +#define GIC_CPU_RUNNINGPRI 0x14 +#define GIC_CPU_HIGHPRI 0x18 +#define GIC_CPU_ALIAS_BINPOINT 0x1c +#define GIC_CPU_ACTIVEPRIO 0xd0 +#define GIC_CPU_IIDR 0xfc +#define GIC_CPU_DIR 0x1000 + +#define GICC_ENABLE 0x1 +#define GICC_INT_PRI_THRESHOLD 0xf0 /* priority levels 16 */ +#define GIC_CPU_CTRL_ENABLE_GRP0 (1 << 0) +#define GIC_CPU_CTRL_ENABLE_GRP1 (1 << 1) +#define GIC_CPU_CTRL_EOI_MODE_NS (1 << 9) + +struct gicv2 +{ + struct rt_pic parent; + + int version; + int max_irq; + + void *dist_base; + rt_size_t dist_size; + void *cpu_base; + rt_size_t cpu_size; + + void *hyp_base; + rt_size_t hyp_size; + void *vcpu_base; + rt_size_t vcpu_size; +}; + +#endif /* __IRQ_GICV2_H__ */ diff --git a/components/drivers/pic/pic-gicv2m.c b/components/drivers/pic/pic-gicv2m.c new file mode 100755 index 000000000000..3aa72dc1190b --- /dev/null +++ b/components/drivers/pic/pic-gicv2m.c @@ -0,0 +1,378 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-11-07 GuEe-GUI first version + */ + +#include +#include +#include + +#define DBG_TAG "pic.gicv2m" +#define DBG_LVL DBG_INFO +#include + +#include +#include +#include "pic-gic-common.h" + +/* +* MSI_TYPER: +* [31:26] Reserved +* [25:16] lowest SPI assigned to MSI +* [15:10] Reserved +* [9:0] Numer of SPIs assigned to MSI +*/ +#define V2M_MSI_TYPER 0x008 +#define V2M_MSI_TYPER_BASE_SHIFT 16 +#define V2M_MSI_TYPER_BASE_MASK 0x3ff +#define V2M_MSI_TYPER_NUM_MASK 0x3ff +#define V2M_MSI_SETSPI_NS 0x040 +#define V2M_MIN_SPI 32 +#define V2M_MAX_SPI 1019 +#define V2M_MSI_IIDR 0xfcc + +#define V2M_MSI_TYPER_BASE_SPI(x) (((x) >> V2M_MSI_TYPER_BASE_SHIFT) & V2M_MSI_TYPER_BASE_MASK) +#define V2M_MSI_TYPER_NUM_SPI(x) ((x) & V2M_MSI_TYPER_NUM_MASK) + +/* APM X-Gene with GICv2m MSI_IIDR register value */ +#define XGENE_GICV2M_MSI_IIDR 0x06000170 +/* Broadcom NS2 GICv2m MSI_IIDR register value */ +#define BCM_NS2_GICV2M_MSI_IIDR 0x0000013f + +/* List of flags for specific v2m implementation */ +#define GICV2M_NEEDS_SPI_OFFSET 0x00000001 +#define GICV2M_GRAVITON_ADDRESS_ONLY 0x00000002 + +struct gicv2m +{ + struct rt_pic parent; + + void *base; + void *base_phy; + rt_uint32_t spi_start; /* The SPI number that MSIs start */ + rt_uint32_t spis_nr; /* The number of SPIs for MSIs */ + rt_uint32_t spi_offset; /* Offset to be subtracted from SPI number */ + + bitmap_t *vectors; /* MSI vector bitmap */ + rt_uint32_t flags; /* Flags for v2m's specific implementation */ + + void *gic; + struct rt_spinlock lock; +}; + +#define raw_to_gicv2m(raw) rt_container_of(raw, struct gicv2m, parent) + +static rt_ubase_t gicv2m_get_msi_addr(struct gicv2m *v2m, int hwirq) +{ + rt_ubase_t addr; + + if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY) + { + addr = (rt_ubase_t)v2m->base_phy | ((hwirq - 32) << 3); + } + else + { + addr = (rt_ubase_t)v2m->base_phy + V2M_MSI_SETSPI_NS; + } + + return addr; +} + +static rt_bool_t is_msi_spi_valid(rt_uint32_t base, rt_uint32_t num) +{ + if (base < V2M_MIN_SPI) + { + LOG_E("Invalid MSI base SPI (base: %u)", base); + + return RT_FALSE; + } + else if ((num == 0) || (base + num > V2M_MAX_SPI)) + { + LOG_E("Number of SPIs (%u) exceed maximum (%u)", num, V2M_MAX_SPI - V2M_MIN_SPI + 1); + + return RT_FALSE; + } + + return RT_TRUE; +} + +static void gicv2m_irq_mask(struct rt_pic_irq *pirq) +{ + rt_pci_msi_mask_irq(pirq); + rt_pic_irq_parent_mask(pirq); +} + +static void gicv2m_irq_unmask(struct rt_pic_irq *pirq) +{ + rt_pci_msi_unmask_irq(pirq); + rt_pic_irq_parent_unmask(pirq); +} + +static void gicv2m_compose_msi_msg(struct rt_pic_irq *pirq, struct rt_pci_msi_msg *msg) +{ + rt_ubase_t addr; + struct gicv2m *v2m = raw_to_gicv2m(pirq->pic); + + addr = gicv2m_get_msi_addr(v2m, pirq->hwirq); + + msg->address_hi = rt_upper_32_bits(addr); + msg->address_lo = rt_lower_32_bits(addr); + + if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY) + { + msg->data = 0; + } + else + { + msg->data = pirq->hwirq; + } + + if (v2m->flags & GICV2M_NEEDS_SPI_OFFSET) + { + msg->data -= v2m->spi_offset; + } +} + +static int gicv2m_irq_alloc_msi(struct rt_pic *pic, struct rt_pci_msi_desc *msi_desc) +{ + rt_ubase_t level; + int irq, parent_irq, hwirq, hwirq_index; + struct rt_pic_irq *pirq; + struct gicv2m *v2m = raw_to_gicv2m(pic); + struct rt_pic *ppic = v2m->gic; + + level = rt_spin_lock_irqsave(&v2m->lock); + hwirq_index = bitmap_next_clear_bit(v2m->vectors, 0, v2m->spis_nr); + + if (hwirq_index >= v2m->spis_nr) + { + irq = -RT_EEMPTY; + goto _out_lock; + } + + hwirq = v2m->spi_start + v2m->spi_offset + hwirq_index; + + parent_irq = ppic->ops->irq_map(ppic, hwirq, RT_IRQ_MODE_EDGE_RISING); + if (parent_irq < 0) + { + irq = parent_irq; + goto _out_lock; + } + + irq = rt_pic_config_irq(pic, hwirq_index, hwirq); + if (irq < 0) + { + goto _out_lock; + } + pirq = rt_pic_find_irq(pic, hwirq_index); + + pirq->mode = RT_IRQ_MODE_EDGE_RISING; + rt_pic_cascade(pirq, parent_irq); + + bitmap_set_bit(v2m->vectors, hwirq_index); + +_out_lock: + rt_spin_unlock_irqrestore(&v2m->lock, level); + + return irq; +} + +static void gicv2m_irq_free_msi(struct rt_pic *pic, int irq) +{ + rt_ubase_t level; + struct rt_pic_irq *pirq; + struct gicv2m *v2m = raw_to_gicv2m(pic); + + pirq = rt_pic_find_pirq(pic, irq); + + if (!pirq) + { + return; + } + + rt_pic_uncascade(pirq); + + level = rt_spin_lock_irqsave(&v2m->lock); + bitmap_clear_bit(v2m->vectors, pirq->hwirq - (v2m->spi_start + v2m->spi_offset)); + rt_spin_unlock_irqrestore(&v2m->lock, level); +} + +static rt_err_t gicv2m_irq_set_state(struct rt_pic *pic, int hwirq, int type, rt_bool_t state) +{ + struct gicv2m *v2m = raw_to_gicv2m(pic); + struct rt_pic *ppic = v2m->gic; + + return ppic->ops->irq_set_state(ppic, hwirq, type, state); +} + +static rt_err_t gicv2m_irq_get_state(struct rt_pic *pic, int hwirq, int type, rt_bool_t *out_state) +{ + struct gicv2m *v2m = raw_to_gicv2m(pic); + struct rt_pic *ppic = v2m->gic; + + return ppic->ops->irq_get_state(ppic, hwirq, type, out_state); +} + +const static struct rt_pic_ops gicv2m_ops = +{ + .name = "GICv2m", + .irq_ack = rt_pic_irq_parent_ack, + .irq_mask = gicv2m_irq_mask, + .irq_unmask = gicv2m_irq_unmask, + .irq_eoi = rt_pic_irq_parent_eoi, + .irq_set_priority = rt_pic_irq_parent_set_priority, + .irq_set_affinity = rt_pic_irq_parent_set_affinity, + .irq_compose_msi_msg = gicv2m_compose_msi_msg, + .irq_alloc_msi = gicv2m_irq_alloc_msi, + .irq_free_msi = gicv2m_irq_free_msi, + .irq_set_state = gicv2m_irq_set_state, + .irq_get_state = gicv2m_irq_get_state, + .flags = RT_PIC_F_IRQ_ROUTING, +}; + +static const struct rt_ofw_node_id gicv2m_ofw_match[] = +{ + { .compatible = "arm,gic-v2m-frame" }, + { /* sentinel */ } +}; + +rt_err_t gicv2m_ofw_probe(struct rt_ofw_node *np, const struct rt_ofw_node_id *id) +{ + rt_err_t err = RT_EOK; + struct rt_ofw_node *v2m_np; + + rt_ofw_foreach_available_child_node(np, v2m_np) + { + struct gicv2m *v2m; + rt_size_t bitmap_size; + rt_uint32_t spi_start = 0, spis_nr = 0; + + if (!rt_ofw_node_match(v2m_np, gicv2m_ofw_match)) + { + continue; + } + + if (!rt_ofw_prop_read_bool(v2m_np, "msi-controller")) + { + continue; + } + + if (!(v2m = rt_malloc(sizeof(*v2m)))) + { + rt_ofw_node_put(v2m_np); + + err = -RT_ENOMEM; + break; + } + + v2m->base = rt_ofw_iomap(v2m_np, 0); + + if (!v2m->base) + { + LOG_E("%s: IO map failed", rt_ofw_node_full_name(v2m_np)); + continue; + } + + v2m->base_phy = rt_kmem_v2p(v2m->base); + v2m->flags = 0; + + if (!rt_ofw_prop_read_u32(v2m_np, "arm,msi-base-spi", &spi_start) && + !rt_ofw_prop_read_u32(v2m_np, "arm,msi-num-spis", &spis_nr)) + { + LOG_I("DT overriding V2M MSI_TYPER (base:%u, num:%u)", spi_start, spis_nr); + } + + if (spi_start && spis_nr) + { + v2m->spi_start = spi_start; + v2m->spis_nr = spis_nr; + } + else + { + rt_uint32_t typer; + + /* Graviton should always have explicit spi_start/spis_nr */ + if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY) + { + goto _fail; + } + typer = HWREG32(v2m->base + V2M_MSI_TYPER); + + v2m->spi_start = V2M_MSI_TYPER_BASE_SPI(typer); + v2m->spis_nr = V2M_MSI_TYPER_NUM_SPI(typer); + } + + if (!is_msi_spi_valid(v2m->spi_start, v2m->spis_nr)) + { + goto _fail; + } + + /* + * APM X-Gene GICv2m implementation has an erratum where + * the MSI data needs to be the offset from the spi_start + * in order to trigger the correct MSI interrupt. This is + * different from the standard GICv2m implementation where + * the MSI data is the absolute value within the range from + * spi_start to (spi_start + num_spis). + * + * Broadcom NS2 GICv2m implementation has an erratum where the MSI data + * is 'spi_number - 32' + * + * Reading that register fails on the Graviton implementation + */ + if (!(v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY)) + { + switch (HWREG32(v2m->base + V2M_MSI_IIDR)) + { + case XGENE_GICV2M_MSI_IIDR: + v2m->flags |= GICV2M_NEEDS_SPI_OFFSET; + v2m->spi_offset = v2m->spi_start; + break; + + case BCM_NS2_GICV2M_MSI_IIDR: + v2m->flags |= GICV2M_NEEDS_SPI_OFFSET; + v2m->spi_offset = 32; + break; + } + } + + bitmap_size = BITMAP_LEN(v2m->spis_nr) * sizeof(bitmap_t); + + if (!(v2m->vectors = rt_calloc(1, bitmap_size))) + { + err = -RT_ENOMEM; + goto _fail; + } + + rt_spin_lock_init(&v2m->lock); + + v2m->parent.priv_data = v2m; + v2m->parent.ops = &gicv2m_ops; + v2m->gic = rt_ofw_data(np); + + rt_pic_linear_irq(&v2m->parent, v2m->spis_nr); + rt_pic_user_extends(&v2m->parent); + + rt_ofw_data(v2m_np) = &v2m->parent; + rt_ofw_node_set_flag(v2m_np, RT_OFW_F_READLY); + + continue; + + _fail: + rt_iounmap(v2m->base); + rt_free(v2m); + + if (err) + { + rt_ofw_node_put(v2m_np); + break; + } + } + + return err; +} diff --git a/components/drivers/pic/pic-gicv3-its.c b/components/drivers/pic/pic-gicv3-its.c new file mode 100755 index 000000000000..ba7a6c3e5a31 --- /dev/null +++ b/components/drivers/pic/pic-gicv3-its.c @@ -0,0 +1,1529 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-01-30 GuEe-GUI first version + */ + +#include +#include +#include + +#define DBG_TAG "pic.gic-v3-its" +#define DBG_LVL DBG_INFO +#include + +#include +#include +#include +#include +#include "pic-gicv3.h" +#include "pic-gic-common.h" + +#define ITS_CMD_QUEUE_SIZE (64 * SIZE_KB) +#define ITS_CMD_QUEUE_ALIGN (64 * SIZE_KB) +#define ITS_CMD_QUEUE_NR (ITS_CMD_QUEUE_SIZE / sizeof(struct its_command)) + +#define ITS_ITT_ALIGN (256 * SIZE_KB) + +#define ITS_MAX_LPI_NIRQS (64 * SIZE_KB) +#define ITS_MAX_LPI_NRBITS 16 /* 64K => 1 << 16 */ +#define ITS_LPI_CONFIG_TABLE_ALIGN (64 * SIZE_KB) +#define ITS_LPI_CONFIG_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI +#define ITS_LPI_CONFIG_PROP_SHIFT 2 +#define ITS_LPI_CONFIG_PROP_MASK RT_GENMASK(7, ITS_LPI_CONFIG_PROP_SHIFT) + +#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1ULL << 0) +#define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1ULL << 1) +#define RDIST_FLAGS_FORCE_NON_SHAREABLE (1ULL << 2) + +#define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0) +#define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1) +#define ITS_FLAGS_FORCE_NON_SHAREABLE (1ULL << 2) + +#define RD_LOCAL_LPI_ENABLED RT_BIT(0) +#define RD_LOCAL_PENDTABLE_PREALLOCATED RT_BIT(1) +#define RD_LOCAL_MEMRESERVE_DONE RT_BIT(2) + +struct its_command +{ + union + { + rt_le64_t code_raw[4]; + rt_uint64_t code[4]; + }; +}; + +struct its_table +{ + void *base; + rt_uint64_t val; + rt_uint32_t size_bits; + rt_uint32_t page_size; + union + { + struct + { + rt_uint32_t itt_entries; + rt_uint32_t lvl2_bits; + }; + }; +}; + +struct its_collection +{ + rt_uint64_t target_address; + rt_uint16_t id; +}; + +struct gicv3_its; + +struct its_map +{ + rt_list_t list; + struct ref ref; + struct gicv3_its *its; + + int device_id; + int lpi_base; + int cpu_id; + + void *itt; + void *lvl2_dte; +}; + +struct gicv3_its +{ + struct rt_pic parent; + rt_list_t list; + + void *base; + void *base_phy; + + void *cmd_base; + rt_ubase_t cmd_idx; + rt_uint32_t flags; + struct rt_spinlock cmd_lock; + + struct its_table tbls[GITS_BASER_NR_REGS]; + struct its_collection collections[RT_CPUS_NR]; + + struct gicv3 *gic; + struct rt_ofw_node *np; +}; + +#define raw_to_gicv3_its(raw) rt_container_of(raw, struct gicv3_its, parent) + +static rt_size_t lpi_nr; +static void *lpi_table; +static void *lpi_pending_table; +static bitmap_t *lpis_vectors = RT_NULL; +static struct rt_spinlock lpis_lock = {}, map_lock = {}; +static rt_list_t its_nodes = RT_LIST_OBJECT_INIT(its_nodes); +static rt_list_t map_nodes = RT_LIST_OBJECT_INIT(map_nodes); + +rt_inline rt_uint64_t its_readq(struct gicv3_its *its, int off) +{ + return HWREG32(its->base + off) | + (rt_uint64_t)HWREG32(its->base + off + 4) << 32; +} + +rt_inline void its_writeq(struct gicv3_its *its, int off, rt_uint64_t value) +{ + HWREG32(its->base + off) = (rt_uint32_t)value; + HWREG32(its->base + off + 4) = (rt_uint32_t)(value >> 32); +} + +rt_inline rt_uint32_t its_readl(struct gicv3_its *its, int off) +{ + return HWREG32(its->base + off); +} + +rt_inline void its_writel(struct gicv3_its *its, int off, rt_uint32_t value) +{ + HWREG32(its->base + off) = value; +} + +rt_inline rt_uint32_t its_pirq_event_id(struct gicv3_its *its, struct rt_pic_irq *pirq) +{ + return pirq->hwirq - 8192; +} + +rt_inline rt_uint32_t its_pirq_device_id(struct gicv3_its *its, struct rt_pic_irq *pirq) +{ + struct its_map *map = pirq->msi_desc->priv; + + return map->device_id; +} + +rt_inline rt_size_t its_device_id_bits(struct gicv3_its *its) +{ + return RT_FIELD_GET(GITS_TYPER_DEVBITS, HWREG64(its->base + GITS_TYPER)) + 1; +} + +rt_inline void *lpi_base_config(int index) +{ + return &((rt_uint8_t *)lpi_table)[index - 8192]; +} + +static void its_mask_encode(rt_uint64_t *raw_code, rt_uint64_t val, int h, int l) +{ + rt_uint64_t mask = RT_GENMASK_ULL(h, l); + *raw_code &= ~mask; + *raw_code |= (val << l) & mask; +} + +rt_inline void its_encode_cmd(struct its_command *cmd, rt_uint8_t cmd_nr) +{ + its_mask_encode(&cmd->code[0], cmd_nr, 7, 0); +} + +rt_inline void its_encode_valid(struct its_command *cmd, rt_bool_t valid) +{ + its_mask_encode(&cmd->code[2], !!valid, 63, 63); +} + +rt_inline void its_encode_phys_id(struct its_command *cmd, rt_uint32_t phys_id) +{ + its_mask_encode(&cmd->code[1], phys_id, 63, 32); +} + +rt_inline void its_encode_size(struct its_command *cmd, rt_uint8_t size) +{ + its_mask_encode(&cmd->code[1], size, 4, 0); +} + +rt_inline void its_encode_itt(struct its_command *cmd, rt_uint64_t itt_addr) +{ + its_mask_encode(&cmd->code[2], itt_addr >> 8, 51, 8); +} + +rt_inline void its_encode_target(struct its_command *cmd, rt_uint64_t target_addr) +{ + its_mask_encode(&cmd->code[2], target_addr >> 16, 51, 16); +} + +rt_inline void its_encode_device_id(struct its_command *cmd, rt_uint32_t device_id) +{ + its_mask_encode(&cmd->code[0], device_id, 63, 32); +} + +rt_inline void its_encode_event_id(struct its_command *cmd, rt_uint32_t event_id) +{ + its_mask_encode(&cmd->code[1], event_id, 31, 0); +} + +rt_inline void its_encode_collection(struct its_command *cmd, rt_uint16_t collection_id) +{ + its_mask_encode(&cmd->code[2], collection_id, 15, 0); +} + +static struct its_table *its_baser_type(struct gicv3_its *its, int type) +{ + for (int i = 0; i < RT_ARRAY_SIZE(its->tbls); ++i) + { + if (GITS_BASER_TYPE(its->tbls[i].val) == type) + { + return &its->tbls[i]; + } + } + + return RT_NULL; +} + +static struct its_command *its_cmd_alloc(struct gicv3_its *its) +{ + struct its_command *cmd = RT_NULL; + + for (rt_uint32_t count = 0; count <= 10000; ++count) + { + if ((its->cmd_idx + 1) % ITS_CMD_QUEUE_NR != its_readl(its, GITS_CREADR) / sizeof(*cmd)) + { + struct its_command *cmds = its->cmd_base; + + cmd = &cmds[its->cmd_idx++]; + its->cmd_idx %= ITS_CMD_QUEUE_NR; + + rt_memset(cmd, 0, sizeof(*cmd)); + + break; + } + + rt_hw_us_delay(10); + } + + return cmd; +} + +static rt_err_t its_cmd_submit_raw(struct gicv3_its *its, struct its_command *cmd) +{ + rt_uint64_t cwriter; + rt_err_t err = -RT_ETIMEOUT; + + cwriter = (void *)(cmd + 1) - its->cmd_base; + rt_hw_rmb(); + +#ifdef ARCH_CPU_BIG_ENDIAN + cmd->code_raw[0] = rt_cpu_to_le64(cmd->code[0]); + cmd->code_raw[1] = rt_cpu_to_le64(cmd->code[1]); + cmd->code_raw[2] = rt_cpu_to_le64(cmd->code[2]); + cmd->code_raw[3] = rt_cpu_to_le64(cmd->code[3]); +#endif /* ARCH_CPU_BIG_ENDIAN */ + + /* Make sure the commands written to memory are observable by the ITS */ + if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) + { + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cmd, sizeof(*cmd)); + } + else + { + rt_hw_wmb(); + } + + its_writel(its, GITS_CWRITER, cwriter); + + for (rt_uint32_t count = 0; count < 10000; ++count) + { + if (its_readl(its, GITS_CREADR) == cwriter) + { + err = RT_EOK; + break; + } + + rt_hw_us_delay(10); + } + + return err; +} + +static rt_err_t its_cmd_submit_nomap(struct gicv3_its *its, struct its_command *cmd, + int cpu_id, rt_bool_t sync) +{ + rt_err_t err; + struct its_command *hw_cmd; + + rt_hw_spin_lock(&its->cmd_lock.lock); + + if (!(hw_cmd = its_cmd_alloc(its))) + { + err = -RT_EBUSY; + goto _out_lock; + } + + rt_memcpy(hw_cmd, cmd, sizeof(*hw_cmd)); + + if ((err = its_cmd_submit_raw(its, hw_cmd))) + { + goto _out_lock; + } + + if (sync) + { + if (!(hw_cmd = its_cmd_alloc(its))) + { + err = -RT_EBUSY; + goto _out_lock; + } + + its_encode_cmd(hw_cmd, GITS_CMD_SYNC); + its_encode_target(hw_cmd, its->collections[cpu_id].target_address); + + err = its_cmd_submit_raw(its, hw_cmd); + } + +_out_lock: + rt_hw_spin_unlock(&its->cmd_lock.lock); + + return err; +} + +static rt_err_t its_cmd_submit(struct gicv3_its *its, struct its_command *cmd, + struct its_map *map, rt_bool_t sync) +{ + return its_cmd_submit_nomap(its, cmd, map->cpu_id, sync); +} + +static rt_err_t lpi_flush_config(struct gicv3_its *its, rt_uint8_t *conf, + struct rt_pic_irq *pirq) +{ + struct its_command cmd; + struct its_map *map = pirq->msi_desc->priv; + + if ((its->gic->redist_flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)) + { + /* Clean D-cache under command */ + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, conf, sizeof(*conf)); + } + else + { + /* DSB inner shareable, store */ + rt_hw_wmb(); + } + + rt_memset(&cmd, 0, sizeof(cmd)); + its_encode_cmd(&cmd, GITS_CMD_INV); + its_encode_device_id(&cmd, its_pirq_device_id(its, pirq)); + its_encode_event_id(&cmd, its_pirq_event_id(its, pirq)); + + return its_cmd_submit(its, &cmd, map, RT_FALSE); +} + +rt_inline void *gicr_rd_base_percpu(struct gicv3 *gic) +{ + return gic->redist_regions[rt_hw_cpu_id()].base; +} + +rt_inline void *gicr_rd_base(struct gicv3_its *its) +{ + return its->gic->redist_percpu_base[rt_hw_cpu_id()]; +} + +rt_inline rt_uint64_t *gicr_rd_flags(struct gicv3_its *its) +{ + return &its->gic->redist_percpu_flags[rt_hw_cpu_id()]; +} + +static rt_bool_t gicr_supports_plpis(struct gicv3_its *its) +{ + return !!(HWREG64(gicr_rd_base(its) + GICR_TYPER) & GICR_TYPER_PLPIS); +} + +static rt_err_t redist_disable_lpis(struct gicv3_its *its) +{ + void *gicr = gicr_rd_base(its); + rt_uint64_t timeout = 1000000L, val; + + if (!gicr_supports_plpis(its)) + { + LOG_E("CPU#%d: LPIs not supported", rt_hw_cpu_id()); + return -RT_ENOSYS; + } + + val = HWREG32(gicr + GICR_CTLR); + if (!(val & GICR_CTLR_ENABLE_LPIS)) + { + return RT_EOK; + } + + /* + * If coming via a CPU hotplug event, we don't need to disable + * LPIs before trying to re-enable them. They are already + * configured and all is well in the world. + * + * If running with preallocated tables, there is nothing to do. + */ + if ((*gicr_rd_flags(its) & RD_LOCAL_LPI_ENABLED) || + (its->gic->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED)) + { + return RT_EOK; + } + + /* From that point on, we only try to do some damage control */ + LOG_W("CPU%d: Booted with LPIs enabled, memory probably corrupted", rt_hw_cpu_id()); + + /* Disable LPIs */ + val &= ~GICR_CTLR_ENABLE_LPIS; + HWREG32(gicr + GICR_CTLR) = val; + + /* Make sure any change to GICR_CTLR is observable by the GIC */ + rt_hw_barrier(dsb, sy); + + /* + * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs + * from 1 to 0 before programming GICR_PEND{PROP}BASER registers. + * Error out if we time out waiting for RWP to clear. + */ + while (HWREG32(gicr + GICR_CTLR) & GICR_CTLR_RWP) + { + if (!timeout) + { + LOG_E("CPU#%d: Timeout while disabling LPIs", rt_hw_cpu_id()); + + return -RT_ETIMEOUT; + } + + rt_hw_us_delay(1); + --timeout; + } + + /* + * After it has been written to 1, it is IMPLEMENTATION + * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be + * cleared to 0. Error out if clearing the bit failed. + */ + if (HWREG32(gicr + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) + { + LOG_E("CPU#%d: Failed to disable LPIs", rt_hw_cpu_id()); + + return -RT_EBUSY; + } + + return RT_EOK; +} + +static void gicv3_its_cpu_init_lpis(struct gicv3_its *its) +{ + void *gicr; + rt_ubase_t paddr; + rt_uint64_t val, tmp; + + if (*gicr_rd_flags(its) & RD_LOCAL_LPI_ENABLED) + { + return; + } + + gicr = gicr_rd_base(its); + + val = HWREG32(gicr + GICR_CTLR); + + if ((its->gic->redist_flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) && + (val & GICR_CTLR_ENABLE_LPIS)) + { + *gicr_rd_flags(its) |= RD_LOCAL_PENDTABLE_PREALLOCATED; + + goto _out; + } + + paddr = (rt_ubase_t)rt_kmem_v2p(lpi_pending_table); + + /* Set PROPBASE */ + val = ((rt_ubase_t)rt_kmem_v2p(lpi_table) | + GITS_CBASER_InnerShareable | + GITS_CBASER_RaWaWb | + ((ITS_MAX_LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK)); + + HWREG64(gicr + GICR_PROPBASER) = val; + tmp = HWREG64(gicr + GICR_PROPBASER); + + if (its->gic->redist_flags & RDIST_FLAGS_FORCE_NON_SHAREABLE) + { + tmp &= ~GICR_PBASER_SHARE_MASK_ALL; + } + + if ((tmp ^ val) & GICR_PBASER_SHARE_MASK_ALL) + { + if (!(tmp & GICR_PBASER_SHARE_MASK_ALL)) + { + /* The HW reports non-shareable, we must remove the cacheability attributes as well */ + val &= ~(GICR_PBASER_SHARE_MASK_ALL | GICR_PBASER_INNER_MASK_ALL); + val |= GICR_PBASER_nC; + HWREG64(gicr + GICR_PROPBASER) = val; + } + + LOG_I("Using cache flushing for LPI property table"); + its->gic->redist_flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; + } + + val = (paddr | GICR_PBASER_InnerShareable | GICR_PBASER_RaWaWb); + + HWREG64(gicr + GICR_PENDBASER) = val; + tmp = HWREG64(gicr + GICR_PENDBASER); + + if (its->gic->redist_flags & RDIST_FLAGS_FORCE_NON_SHAREABLE) + { + tmp &= ~GICR_PBASER_SHARE_MASK_ALL; + } + + if (!(tmp & GICR_PBASER_SHARE_MASK_ALL)) + { + /* + * The HW reports non-shareable, we must remove the + * cacheability attributes as well. + */ + val &= ~(GICR_PBASER_SHARE_MASK_ALL | GICR_PBASER_INNER_MASK_ALL); + val |= GICR_PBASER_nC; + HWREG64(gicr + GICR_PENDBASER) = val; + } + + /* Enable LPIs */ + val = HWREG32(gicr + GICR_CTLR); + val |= GICR_CTLR_ENABLE_LPIS; + HWREG32(gicr + GICR_CTLR) = val; + + rt_hw_barrier(dsb, sy); + +_out: + *gicr_rd_flags(its) |= RD_LOCAL_LPI_ENABLED; +} + +static void gicv3_its_cpu_init_collection(struct gicv3_its *its) +{ + rt_uint64_t target; + int cpu_id = rt_hw_cpu_id(); + struct its_command cmd; + struct its_collection *collection; + + if (HWREG64(its->base + GITS_TYPER) & GITS_TYPER_PTA) + { + target = (rt_uint64_t)rt_kmem_v2p(gicr_rd_base(its)); + } + else + { + /* Linear by GICR processor number */ + target = HWREG64(gicr_rd_base(its) + GICR_TYPER); + target = GICR_TYPER_CPU_NO(target) << 16; + } + + collection = &its->collections[cpu_id]; + collection->target_address = target; + collection->id = cpu_id; + + rt_memset(&cmd, 0, sizeof(cmd)); + its_encode_cmd(&cmd, GITS_CMD_MAPC); + its_encode_collection(&cmd, collection->id); + its_encode_target(&cmd, target); + its_encode_valid(&cmd, RT_TRUE); + its_cmd_submit_nomap(its, &cmd, cpu_id, RT_TRUE); + + rt_memset(&cmd, 0, sizeof(cmd)); + its_encode_cmd(&cmd, GITS_CMD_INVALL); + its_encode_collection(&cmd, collection->id); + its_cmd_submit_nomap(its, &cmd, cpu_id, RT_TRUE); +} + +static rt_err_t gicv3_its_irq_init(struct rt_pic *pic) +{ + rt_err_t err; + struct gicv3_its *its = raw_to_gicv3_its(pic); + + if ((err = redist_disable_lpis(its))) + { + return err; + } + + gicv3_its_cpu_init_lpis(its); + gicv3_its_cpu_init_collection(its); + + return RT_EOK; +} + +static void gicv3_its_irq_mask(struct rt_pic_irq *pirq) +{ + rt_uint8_t *conf = lpi_base_config(pirq->hwirq); + struct gicv3_its *its = raw_to_gicv3_its(pirq->pic); + + *conf &= ~GITS_LPI_CFG_ENABLED; + lpi_flush_config(its, conf, pirq); + + rt_pci_msi_mask_irq(pirq); +} + +static void gicv3_its_irq_unmask(struct rt_pic_irq *pirq) +{ + rt_uint8_t *conf = lpi_base_config(pirq->hwirq); + struct gicv3_its *its = raw_to_gicv3_its(pirq->pic); + + *conf |= GITS_LPI_CFG_ENABLED; + lpi_flush_config(its, conf, pirq); + + rt_pci_msi_unmask_irq(pirq); +} + +static rt_err_t gicv3_its_irq_set_priority(struct rt_pic_irq *pirq, rt_uint32_t priority) +{ + rt_uint8_t *conf = lpi_base_config(pirq->hwirq); + struct gicv3_its *its = raw_to_gicv3_its(pirq->pic); + + *conf = (priority << ITS_LPI_CONFIG_PROP_SHIFT) | (*conf & (~ITS_LPI_CONFIG_PROP_MASK)); + + return lpi_flush_config(its, conf, pirq); +} + +static rt_err_t gicv3_its_irq_set_affinity(struct rt_pic_irq *pirq, bitmap_t *affinity) +{ + int cpu_id; + rt_err_t err; + struct its_map *map; + struct its_command cmd; + struct its_collection *collection; + struct gicv3_its *its = raw_to_gicv3_its(pirq->pic); + + map = pirq->msi_desc->priv; + cpu_id = bitmap_next_set_bit(affinity, 0, RT_CPUS_NR); + collection = &its->collections[cpu_id]; + + if (rt_unlikely(collection->target_address == ~0ULL)) + { + return -RT_EIO; + } + + if (map->cpu_id == cpu_id) + { + return RT_EOK; + } + + rt_memset(&cmd, 0, sizeof(cmd)); + its_encode_cmd(&cmd, GITS_CMD_MOVI); + its_encode_device_id(&cmd, map->device_id); + its_encode_event_id(&cmd, its_pirq_event_id(its, pirq)); + its_encode_collection(&cmd, collection->id); + + if (!(err = its_cmd_submit(its, &cmd, map, RT_TRUE))) + { + map->cpu_id = cpu_id; + } + + return err; +} + +static void gicv3_its_irq_compose_msi_msg(struct rt_pic_irq *pirq, struct rt_pci_msi_msg *msg) +{ + rt_ubase_t addr; + struct gicv3_its *its = raw_to_gicv3_its(pirq->pic); + + addr = (rt_ubase_t)its->base_phy + GITS_TRANSLATER; + + msg->address_hi = rt_upper_32_bits(addr); + msg->address_lo = rt_lower_32_bits(addr); + msg->data = its_pirq_event_id(its, pirq); +} + +static int gicv3_its_irq_alloc_msi(struct rt_pic *pic, struct rt_pci_msi_desc *msi_desc) +{ + rt_ubase_t level; + rt_uint32_t device_id = -1; + int irq = -1, hwirq, parent_irq, hwirq_index, lpi_base = 0; + struct its_map *map = RT_NULL, *map_tmp; + struct its_table *tbl; + struct its_command cmd; + struct rt_pic_irq *pirq; + struct rt_pci_device *pdev = msi_desc->pdev; + struct gicv3_its *its = raw_to_gicv3_its(pic); + struct rt_pic *ppic = &its->gic->parent; + + tbl = its_baser_type(its, GITS_BASER_TYPE_DEVICE); + RT_ASSERT(tbl != RT_NULL); + + if (!pdev->parent.ofw_node) + { + device_id = pdev->devfn; + } + else + { + struct rt_ofw_cell_args args; + + for (int index = 0; ; ++index) + { + rt_err_t err = rt_ofw_parse_phandle_cells(pdev->parent.ofw_node, + "msi-parent", "#msi-cells", index, &args); + + if (rt_unlikely(err)) + { + return (int)err; + } + + if (args.data == its->np) + { + device_id = args.args[0]; + } + + rt_ofw_node_put(args.data); + + if ((rt_int32_t)device_id >= 0) + { + break; + } + } + } + + if (device_id >= (1 << tbl->size_bits)) + { + LOG_E("Device ID = is %x not supported", device_id); + + return -RT_EINVAL; + } + + /* Find old map info */ + level = rt_spin_lock_irqsave(&map_lock); + rt_list_for_each_entry(map_tmp, &map_nodes, list) + { + if (map_tmp->device_id == device_id) + { + map = map_tmp; + lpi_base = map->lpi_base - 8192; + break; + } + } + rt_spin_unlock_irqrestore(&map_lock, level); + + if (!map) + { + rt_size_t itt_size; + + if (!(map = rt_calloc(1, sizeof(*map)))) + { + return -RT_ENOMEM; + } + + itt_size = tbl->itt_entries * (RT_FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, + HWREG64(its->base + GITS_TYPER)) + 1); + itt_size = rt_max_t(rt_size_t, itt_size, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; + + map->itt = rt_malloc_align(itt_size, ITS_ITT_ALIGN); + + if (!map->itt) + { + rt_free(map); + return -RT_ENOMEM; + } + + if (tbl->lvl2_bits) + { + void *lvl2_dte; + rt_uint64_t *entry; + + entry = tbl->base + (device_id / (tbl->page_size / GITS_LVL1_ENTRY_SIZE)); + + if (*entry) + { + lvl2_dte = (void *)(*entry - PV_OFFSET); + rt_page_ref_inc(lvl2_dte, tbl->lvl2_bits); + } + else + { + lvl2_dte = rt_pages_alloc(tbl->lvl2_bits); + + if (!lvl2_dte) + { + rt_free_align(map->itt); + rt_free(map); + return -RT_ENOMEM; + } + + *entry = rt_cpu_to_le64(GITS_BASER_VALID | (rt_uint64_t)rt_kmem_v2p(lvl2_dte)); + + if (!(tbl->val & GITS_BASER_SHARE_MASK_ALL)) + { + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, entry, sizeof(*entry)); + } + + rt_hw_dsb(); + } + + map->lvl2_dte = lvl2_dte; + } + + rt_memset(map->itt, 0, itt_size); + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, map->itt, itt_size); + } + msi_desc->priv = map; + + /* Alloc the LPI base on the first LPI */ + level = rt_spin_lock_irqsave(&lpis_lock); + hwirq_index = bitmap_next_clear_bit(lpis_vectors, lpi_base, lpi_nr); + + if (hwirq_index >= lpi_nr) + { + irq = -RT_EEMPTY; + goto _out_lock; + } + + hwirq = 8192 + hwirq_index; + parent_irq = ppic->ops->irq_map(ppic, hwirq, RT_IRQ_MODE_EDGE_RISING); + if (parent_irq < 0) + { + irq = parent_irq; + goto _out_lock; + } + + irq = rt_pic_config_irq(pic, hwirq_index, hwirq); + if (irq < 0) + { + goto _out_lock; + } + pirq = rt_pic_find_irq(pic, hwirq_index); + + pirq->mode = RT_IRQ_MODE_EDGE_RISING; + rt_pic_cascade(pirq, parent_irq); + + bitmap_set_bit(lpis_vectors, hwirq_index); + +_out_lock: + rt_spin_unlock_irqrestore(&lpis_lock, level); + + if (irq < 0) + { + return irq; + } + + if (map->its) + { + ref_get(&map->ref); + } + else + { + rt_list_init(&map->list); + ref_init(&map->ref); + map->its = its; + map->device_id = device_id; + map->lpi_base = hwirq; + + level = rt_spin_lock_irqsave(&map_lock); + rt_list_insert_before(&map_nodes, &map->list); + rt_spin_unlock_irqrestore(&map_lock, level); + } + + /* Default to CPU#0 */ + map->cpu_id = 0; + RT_IRQ_AFFINITY_SET(pirq->affinity, map->cpu_id); + + rt_memset(&cmd, 0, sizeof(cmd)); + its_encode_cmd(&cmd, GITS_CMD_MAPD); + its_encode_device_id(&cmd, device_id); + its_encode_size(&cmd, rt_ilog2(tbl->itt_entries) - 1); + its_encode_itt(&cmd, (rt_uint64_t)rt_kmem_v2p(map->itt)); + its_encode_valid(&cmd, RT_TRUE); + its_cmd_submit(its, &cmd, map, RT_FALSE); + + rt_memset(&cmd, 0, sizeof(cmd)); + its_encode_cmd(&cmd, GITS_CMD_MAPTI); + its_encode_device_id(&cmd, device_id); + its_encode_event_id(&cmd, its_pirq_event_id(its, pirq)); + its_encode_phys_id(&cmd, hwirq); + its_encode_collection(&cmd, its->collections[map->cpu_id].id); + its_cmd_submit(its, &cmd, map, RT_TRUE); + + return irq; +} + +static void its_map_release(struct ref *r) +{ + rt_ubase_t level; + struct gicv3_its *its; + struct its_table *tbl; + struct its_command cmd; + struct its_map *map = rt_container_of(r, struct its_map, ref); + + its = map->its; + tbl = its_baser_type(its, GITS_BASER_TYPE_DEVICE); + + rt_memset(&cmd, 0, sizeof(cmd)); + its_encode_cmd(&cmd, GITS_CMD_MAPD); + its_encode_device_id(&cmd, map->device_id); + its_encode_size(&cmd, rt_ilog2(tbl->itt_entries) - 1); + its_encode_itt(&cmd, (rt_uint64_t)rt_kmem_v2p(map->itt)); + its_encode_valid(&cmd, RT_FALSE); + its_cmd_submit(its, &cmd, map, RT_TRUE); + + level = rt_spin_lock_irqsave(&map_lock); + rt_list_insert_before(&map_nodes, &map->list); + rt_spin_unlock_irqrestore(&map_lock, level); + + if (map->itt) + { + rt_free_align(map->itt); + } + if (map->lvl2_dte) + { + if (rt_page_ref_get(map->lvl2_dte, tbl->lvl2_bits) == 1) + { + rt_uint64_t *entry; + + entry = tbl->base + (map->device_id / (tbl->page_size / GITS_LVL1_ENTRY_SIZE)); + *entry = rt_cpu_to_le64(0); + + if (!(tbl->val & GITS_BASER_SHARE_MASK_ALL)) + { + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, entry, sizeof(*entry)); + } + } + rt_pages_free(map->lvl2_dte, tbl->lvl2_bits); + } + rt_free(map); +} + +static void gicv3_its_irq_free_msi(struct rt_pic *pic, int irq) +{ + rt_ubase_t level; + struct its_map *map; + struct its_command cmd; + struct rt_pic_irq *pirq; + struct gicv3_its *its = raw_to_gicv3_its(pic); + + pirq = rt_pic_find_pirq(pic, irq); + + if (!pirq) + { + return; + } + + map = pirq->msi_desc->priv; + + rt_memset(&cmd, 0, sizeof(cmd)); + its_encode_cmd(&cmd, GITS_CMD_DISCARD); + its_encode_device_id(&cmd, map->device_id); + its_encode_event_id(&cmd, its_pirq_event_id(its, pirq)); + its_cmd_submit(its, &cmd, map, RT_TRUE); + + rt_pic_uncascade(pirq); + + level = rt_spin_lock_irqsave(&lpis_lock); + bitmap_clear_bit(lpis_vectors, pirq->hwirq - 8192); + rt_spin_unlock_irqrestore(&lpis_lock, level); + + ref_put(&map->ref, its_map_release); +} + +static rt_err_t gicv3_its_irq_set_state(struct rt_pic *pic, int hwirq, int type, rt_bool_t state) +{ + struct its_map *map; + struct its_command cmd; + struct rt_pic_irq *pirq; + struct gicv3_its *its = raw_to_gicv3_its(pic); + + if (type != RT_IRQ_STATE_PENDING || hwirq > 8192 + lpi_nr) + { + return -RT_ENOSYS; + } + + if (!(pirq = rt_pic_find_irq(pic, hwirq - 8192))) + { + return -RT_ENOSYS; + } + + map = pirq->msi_desc->priv; + rt_memset(&cmd, 0, sizeof(cmd)); + + if (state) + { + its_encode_cmd(&cmd, GITS_CMD_INT); + its_encode_device_id(&cmd, map->device_id); + its_encode_event_id(&cmd, its_pirq_event_id(its, pirq)); + } + else + { + its_encode_cmd(&cmd, GITS_CMD_CLEAR); + its_encode_device_id(&cmd, map->device_id); + its_encode_event_id(&cmd, its_pirq_event_id(its, pirq)); + } + + its_cmd_submit(its, &cmd, map, RT_TRUE); + + return RT_EOK; +} + +const static struct rt_pic_ops gicv3_its_ops = +{ + .name = "GICv3-ITS", + .irq_init = gicv3_its_irq_init, + .irq_ack = rt_pic_irq_parent_ack, + .irq_mask = gicv3_its_irq_mask, + .irq_unmask = gicv3_its_irq_unmask, + .irq_eoi = rt_pic_irq_parent_eoi, + .irq_set_priority = gicv3_its_irq_set_priority, + .irq_set_affinity = gicv3_its_irq_set_affinity, + .irq_compose_msi_msg = gicv3_its_irq_compose_msi_msg, + .irq_alloc_msi = gicv3_its_irq_alloc_msi, + .irq_free_msi = gicv3_its_irq_free_msi, + .irq_set_state = gicv3_its_irq_set_state, + .flags = RT_PIC_F_IRQ_ROUTING, +}; + +static rt_ssize_t its_baser_page_size(struct gicv3_its *its, struct its_table *tbl) +{ + rt_size_t page_size = 64 * SIZE_KB; + + while (page_size) + { + rt_uint64_t val, baser_page_size; + rt_off_t baser = GITS_BASERn((int)(tbl - its->tbls)); + + val = its_readq(its, baser); + val &= ~GITS_BASER_PAGE_SIZE_MASK; + + switch (page_size) + { + case 64 * SIZE_KB: + baser_page_size = GITS_BASER_PAGE_SIZE_64K; + break; + case 16 * SIZE_KB: + baser_page_size = GITS_BASER_PAGE_SIZE_16K; + break; + case 4 * SIZE_KB: + default: + baser_page_size = GITS_BASER_PAGE_SIZE_4K; + break; + } + + baser_page_size >>= GITS_BASER_PAGE_SIZE_SHIFT; + + val |= RT_FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, baser_page_size); + its_writeq(its, baser, val); + tbl->val = its_readq(its, baser); + + if (RT_FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, tbl->val) == baser_page_size) + { + break; + } + + switch (page_size) + { + case 64 * SIZE_KB: + page_size = 16 * SIZE_KB; + break; + case 16 * SIZE_KB: + page_size = 4 * SIZE_KB; + break; + case 4 * SIZE_KB: + default: + return -RT_EINVAL; + } + } + + return page_size; +} + +static rt_err_t its_table_init(struct gicv3_its *its) +{ + int inited = 0; + rt_off_t baser; + rt_bool_t indirect = RT_FALSE; + rt_size_t pages_nr, alloc_size; + rt_uint64_t val, type, entry_size, share, cache; + struct its_table *tbl; + + share = GITS_BASER_InnerShareable; + cache = GITS_BASER_RaWaWb; + + for (int i = 0; i < RT_ARRAY_SIZE(its->tbls); ++i) + { + tbl = &its->tbls[i]; + + val = its_readq(its, GITS_BASERn(i)); + type = GITS_BASER_TYPE(val); + + if (type != GITS_BASER_TYPE_DEVICE && + type != GITS_BASER_TYPE_COLLECTION) + { + continue; + } + + tbl->page_size = its_baser_page_size(its, tbl); + + if (tbl->page_size < 0) + { + continue; + } + + baser = GITS_BASERn((int)(tbl - its->tbls)); + entry_size = GITS_BASER_ENTRY_SIZE(val); + + if (type == GITS_BASER_TYPE_DEVICE) + { + tbl->size_bits = its_device_id_bits(its); + LOG_D("Device Max IDs = %lu", 1UL << tbl->size_bits); + + /* For MSI-X */ + tbl->itt_entries = 2048; + while (MAX_HANDLERS / tbl->itt_entries < (1 << tbl->size_bits) && + tbl->itt_entries > 32) + { + tbl->itt_entries >>= 1; + } + } + + its_writeq(its, baser, tbl->val | GITS_BASER_INDIRECT); + tbl->val = its_readq(its, baser); + + indirect = !!(tbl->val & GITS_BASER_INDIRECT); + if (indirect && type == GITS_BASER_TYPE_DEVICE) + { + /* The size of the level 2 table is equal to ITS page size */ + tbl->lvl2_bits = tbl->size_bits - rt_ilog2(tbl->page_size / (int)entry_size); + + /* Get level 1 entries count */ + alloc_size = (1 << tbl->size_bits) / (tbl->page_size / entry_size); + alloc_size *= GITS_LVL1_ENTRY_SIZE; + } + else + { + alloc_size = (1 << tbl->size_bits) * entry_size; + indirect = RT_FALSE; + } + + tbl->base = rt_malloc_align(alloc_size, tbl->page_size); + pages_nr = alloc_size / tbl->page_size; + + if (!tbl->base) + { + return -RT_ENOMEM; + } + + if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) + { + cache = GITS_BASER_nCnB; + } + + if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) + { + cache = GITS_BASER_nC; + share = 0; + } + + val = ((rt_ubase_t)rt_kmem_v2p(tbl->base) | + (type << GITS_BASER_TYPE_SHIFT) | + ((entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | + (pages_nr << GITS_BASER_PAGES_SHIFT) | + cache | share | GITS_BASER_VALID); + val |= indirect ? GITS_BASER_INDIRECT : 0; + + switch (tbl->page_size) + { + case 4 * SIZE_KB: + val |= GITS_BASER_PAGE_SIZE_4K; + break; + case 16 * SIZE_KB: + val |= GITS_BASER_PAGE_SIZE_16K; + break; + case 64 * SIZE_KB: + val |= GITS_BASER_PAGE_SIZE_64K; + break; + } + + its_writeq(its, baser, val); + tbl->val = its_readq(its, baser); + + rt_memset(tbl->base, 0, alloc_size); + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, tbl->base, alloc_size); + + cache = tbl->val & GITS_BASER_INNER_MASK_ALL; + share = tbl->val & GITS_BASER_SHARE_MASK_ALL; + + ++inited; + } + + return inited == 2 ? RT_EOK : -RT_ENOSYS; +} + +static rt_err_t its_cmd_queue_init(struct gicv3_its *its) +{ + void *cmd_phy_base; + rt_uint64_t baser, tmp; + + its->cmd_base = rt_malloc_align(ITS_CMD_QUEUE_SIZE, ITS_CMD_QUEUE_ALIGN); + + if (!its->cmd_base) + { + return -RT_ENOMEM; + } + + its->cmd_idx = 0; + + cmd_phy_base = rt_kmem_v2p(its->cmd_base); + + baser = GITS_CBASER_VALID | GITS_CBASER_RaWaWb | GITS_CBASER_InnerShareable | \ + ((rt_uint64_t)cmd_phy_base) | (ITS_CMD_QUEUE_SIZE / (4 * SIZE_KB) - 1); + + its_writeq(its, GITS_CBASER, baser); + tmp = its_readq(its, GITS_CBASER); + + if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) + { + tmp &= ~GITS_CBASER_SHARE_MASK_ALL; + } + + if ((tmp ^ baser) & GITS_CBASER_SHARE_MASK_ALL) + { + if (!(tmp & GITS_CBASER_SHARE_MASK_ALL)) + { + /* The HW reports non-shareable, we must remove the cacheability attributes as well */ + baser &= ~(GITS_CBASER_SHARE_MASK_ALL | GITS_CBASER_INNER_MASK_ALL); + baser |= GITS_CBASER_nC; + + its_writeq(its, GITS_CBASER, baser); + } + + LOG_I("Using cache flushing for cmd queue"); + its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; + } + + /* Get the next command from the start of the buffer */ + its_writeq(its, GITS_CWRITER, 0); + + return RT_EOK; +} + +static rt_err_t its_lpi_table_init(struct gicv3 *gic) +{ + rt_size_t lpi_table_size; + rt_uint32_t id_bits, numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic->gicd_typer); + + if (HWREG32(gicr_rd_base_percpu(gic) + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) + { + gic->redist_flags |= + RDIST_FLAGS_RD_TABLES_PREALLOCATED | + RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; + + LOG_I("Using preallocated redistributor tables"); + } + + id_bits = GICD_TYPER_ID_BITS(gic->gicd_typer); + + if (gic->redist_flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) + { + rt_uint64_t val = HWREG64(gicr_rd_base_percpu(gic) + GICR_PROPBASER); + id_bits = rt_min_t(rt_uint32_t, id_bits, (val & GICR_PROPBASER_IDBITS_MASK) + 1); + } + + lpi_nr = rt_min_t(rt_size_t, (1UL << id_bits) - 8192, gic->lpi_nr); + + if (numlpis > 2 && numlpis > lpi_nr) + { + lpi_nr = numlpis; + LOG_W("Using hypervisor restricted LPI range [%u]", lpi_nr); + } + + gic->lpi_nr = lpi_nr; + + lpi_table_size = lpi_nr; /* LPI Configuration table entry is 1 byte */ + lpi_table = rt_malloc_align(lpi_table_size * 2, ITS_LPI_CONFIG_TABLE_ALIGN); + + if (!lpi_table) + { + return -RT_ENOMEM; + } + + if (!(lpis_vectors = rt_calloc(1, BITMAP_LEN(lpi_nr) * sizeof(bitmap_t)))) + { + rt_free_align(lpi_table); + lpi_table = RT_NULL; + + return -RT_ENOMEM; + } + + lpi_pending_table = lpi_table + lpi_table_size; + + /* Set the default configuration */ + rt_memset(lpi_table, ITS_LPI_CONFIG_PROP_DEFAULT_PRIO | GITS_LPI_CFG_GROUP1, lpi_table_size); + + /* Flush the table to memory */ + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, lpi_table, lpi_table_size); + + LOG_D("ITS: Allocator initialized for %u LPIs", lpi_nr); + + return RT_EOK; +} + +static void its_init_fail(struct gicv3_its *its) +{ + if (its->base) + { + rt_iounmap(its->base); + } + + if (its->cmd_base) + { + rt_free_align(its->cmd_base); + } + + for (int i = 0; i < RT_ARRAY_SIZE(its->tbls); ++i) + { + struct its_table *tbl = &its->tbls[i]; + + if (tbl->base) + { + rt_free_align(tbl->base); + } + } + + rt_list_remove(&its->list); + rt_free(its); +} + +static rt_err_t its_quirk_cavium_22375(void *data) +{ + struct gicv3_its *its = data; + + its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; + + return RT_EOK; +} + +static rt_err_t its_enable_rk3588001(void *data) +{ + struct gicv3_its *its = data; + struct gicv3 *gic = its->gic; + + if (!rt_ofw_machine_is_compatible("rockchip,rk3588") && + !rt_ofw_machine_is_compatible("rockchip,rk3588s")) + { + return -RT_EINVAL; + } + + its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; + gic->redist_flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE; + + return RT_EOK; +} + +static rt_err_t its_set_non_coherent(void *data) +{ + struct gicv3_its *its = data; + + if (!rt_ofw_prop_read_bool(its->np, "dma-noncoherent")) + { + return -RT_EINVAL; + } + + its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; + + return RT_EOK; +} + +static const struct gic_quirk _its_quirks[] = +{ + { + .desc = "ITS: Cavium ThunderX errata: 22375, 24313", + .iidr = 0xa100034c, + .iidr_mask = 0xffff0fff, + .init = its_quirk_cavium_22375, + }, + { + .desc = "ITS: Rockchip erratum RK3588001", + .iidr = 0x0201743b, + .iidr_mask = 0xffffffff, + .init = its_enable_rk3588001, + }, + { + .desc = "ITS: non-coherent attribute", + .compatible = "arm,gic-v3-its", + .init = its_set_non_coherent, + }, + { /* sentinel */ } +}; + +static const struct rt_ofw_node_id gicv3_its_ofw_match[] = +{ + { .compatible = "arm,gic-v3-its" }, + { /* sentinel */ } +}; + +rt_err_t gicv3_its_ofw_probe(struct rt_ofw_node *np, const struct rt_ofw_node_id *id) +{ + rt_err_t err = -RT_EEMPTY; + struct rt_ofw_node *its_np; + struct gicv3_its *its, *its_next; + + rt_ofw_foreach_available_child_node(np, its_np) + { + if (!rt_ofw_node_match(its_np, gicv3_its_ofw_match)) + { + continue; + } + + if (!rt_ofw_prop_read_bool(its_np, "msi-controller")) + { + continue; + } + + if (!(its = rt_malloc(sizeof(struct gicv3_its)))) + { + rt_ofw_node_put(its_np); + + err = -RT_ENOMEM; + goto _free_all; + } + + its->base = rt_ofw_iomap(its_np, 0); + + if (!its->base) + { + LOG_E("%s: IO map failed", rt_ofw_node_full_name(its_np)); + its_init_fail(its); + continue; + } + + /* + * Make sure ALL the ITS are reset before we probe any, + * as they may be sharing memory + */ + for (int i = 0; i < GITS_BASER_NR_REGS; ++i) + { + its_writeq(its, GITS_BASER + (i << 3), 0); + } + + its->np = its_np; + rt_list_init(&its->list); + rt_list_insert_before(&its_nodes, &its->list); + } + + if (!rt_list_isempty(&its_nodes)) + { + if ((err = its_lpi_table_init(rt_ofw_data(np)))) + { + goto _free_all; + } + } + + rt_list_for_each_entry_safe(its, its_next, &its_nodes, list) + { + rt_uint32_t ctlr; + + its->base_phy = rt_kmem_v2p(its->base); + its->gic = rt_ofw_data(np); + + gic_common_init_quirk_hw(HWREG32(its->base + GITS_IIDR), _its_quirks, its); + gic_common_init_quirk_ofw(its->np, _its_quirks, its); + + if ((err = its_cmd_queue_init(its))) + { + goto _fail; + } + rt_spin_lock_init(&its->cmd_lock); + + if ((err = its_table_init(its))) + { + goto _fail; + } + + for (int i = 0; i < RT_CPUS_NR; ++i) + { + its->collections[i].target_address = ~0ULL; + } + + ctlr = its_readl(its, GITS_CTLR); + ctlr |= GITS_CTLR_ENABLE; + its_writel(its, GITS_CTLR, ctlr); + + its->parent.priv_data = its; + its->parent.ops = &gicv3_its_ops; + + rt_pic_linear_irq(&its->parent, its->gic->lpi_nr); + rt_pic_user_extends(&its->parent); + + its_np = its->np; + rt_ofw_data(its_np) = &its->parent; + rt_ofw_node_set_flag(its_np, RT_OFW_F_READLY); + + continue; + + _fail: + its_init_fail(its); + + if (err == -RT_ENOMEM) + { + break; + } + } + + if (rt_list_isempty(&its_nodes) && lpis_vectors) + { + rt_free(lpis_vectors); + rt_free_align(lpi_table); + lpis_vectors = RT_NULL; + } + + return err; + +_free_all: + rt_list_for_each_entry_safe(its, its_next, &its_nodes, list) + { + rt_free(its); + rt_list_remove(&its->list); + } + + return err; +} diff --git a/components/drivers/pic/pic-gicv3.c b/components/drivers/pic/pic-gicv3.c new file mode 100644 index 000000000000..7a6f6840c433 --- /dev/null +++ b/components/drivers/pic/pic-gicv3.c @@ -0,0 +1,1096 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2013-07-20 Bernard first version + * 2014-04-03 Grissiom many enhancements + * 2018-11-22 Jesven add rt_hw_ipi_send() + * add rt_hw_ipi_handler_install() + * 2022-08-24 GuEe-GUI add pic support + * 2022-11-07 GuEe-GUI add v2m support + * 2023-01-30 GuEe-GUI add its and espi, eppi, lpi support + */ + +#include +#include +#include + +#define DBG_TAG "pic.gicv3" +#define DBG_LVL DBG_INFO +#include + +#include + +#include +#include + +#include "pic-gicv3.h" +#include "pic-gic-common.h" + +#define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) + +static int _init_cpu_id; +static struct gicv3 _gic; +static rt_bool_t _gicv3_eoi_mode_ns = RT_FALSE; +static rt_bool_t _gicv3_arm64_2941627_erratum = RT_FALSE; + +enum +{ + SGI_TYPE, + PPI_TYPE, + SPI_TYPE, + EPPI_TYPE, + ESPI_TYPE, + LPI_TYPE, + + UNKNOW_TYPE, +}; + +rt_inline void *gicv3_percpu_redist_base(void) +{ + return _gic.redist_percpu_base[rt_hw_cpu_id()]; +} + +rt_inline void *gicv3_percpu_redist_sgi_base(void) +{ + return gicv3_percpu_redist_base() + GICR_SGI_OFFSET; +} + +static rt_uint16_t *gicv3_dist_espi_reg(rt_uint32_t offset) +{ +#define __reg_map_bits 5 +#define __reg_map_size (1 << __reg_map_bits) + static rt_uint16_t reg_map[__reg_map_size] = {}; + + int idx = hashmap_32(offset, __reg_map_bits); + + LOG_D("%s ESPI Map<0x%04x> = %2d", "Distributor", offset, idx); + + return ®_map[idx]; +#undef __reg_map_bits +#undef __reg_map_size +} + +static void gicv3_wait_for_rwp(void *base, rt_uint32_t rwp_bit) +{ + rt_uint32_t count = 1000000; + + while ((HWREG32(base + GICD_CTLR) & rwp_bit)) + { + count--; + if (!count) + { + LOG_W("RWP timeout"); + break; + } + + rt_hw_cpu_relax(); + } +} + +rt_inline void gicv3_dist_wait_for_rwp(void) +{ + gicv3_wait_for_rwp(_gic.dist_base, GICD_CTLR_RWP); +} + +rt_inline void gicv3_redist_wait_for_rwp(void) +{ + gicv3_wait_for_rwp(_gic.redist_percpu_base[rt_hw_cpu_id()], GICR_CTLR_RWP); +} + +static typeof(UNKNOW_TYPE) gicv3_hwirq_type(int hwirq) +{ + typeof(UNKNOW_TYPE) ret; + + switch (hwirq) + { + case 0 ... 15: + ret = SGI_TYPE; + break; + case 16 ... 31: + ret = PPI_TYPE; + break; + case 32 ... 1019: + ret = SPI_TYPE; + break; + case GIC_EPPI_BASE_INTID ... (GIC_EPPI_BASE_INTID + 63): + ret = EPPI_TYPE; + break; + case GIC_ESPI_BASE_INTID ... (GIC_ESPI_BASE_INTID + 1023): + ret = ESPI_TYPE; + break; + case 8192 ... RT_GENMASK(23, 0): + ret = LPI_TYPE; + break; + default: + ret = UNKNOW_TYPE; + break; + } + + return ret; +} + +static rt_uint32_t gicv3_hwirq_convert_offset_index(int hwirq, rt_uint32_t offset, rt_uint32_t *index) +{ + switch (gicv3_hwirq_type(hwirq)) + { + case SGI_TYPE: + case PPI_TYPE: + case SPI_TYPE: + *index = hwirq; + break; + case EPPI_TYPE: + /* EPPI range (GICR_IPRIORITYRE) is contiguousto the PPI (GICR_IPRIORITYR) range in the registers */ + *index = hwirq - GIC_EPPI_BASE_INTID + 32; + break; + case ESPI_TYPE: + *index = hwirq - GIC_ESPI_BASE_INTID; + offset = *gicv3_dist_espi_reg(offset); + break; + default: + *index = hwirq; + break; + } + + return offset; +} + +rt_inline rt_bool_t gicv3_hwirq_in_redist(int hwirq) +{ + switch (gicv3_hwirq_type(hwirq)) + { + case SGI_TYPE: + case PPI_TYPE: + case EPPI_TYPE: + return RT_TRUE; + default: + return RT_FALSE; + } +} + +static void *gicv3_hwirq_reg_base(int hwirq, rt_uint32_t offset, rt_uint32_t *index) +{ + void *base; + + if (gicv3_hwirq_in_redist(hwirq)) + { + base = gicv3_percpu_redist_sgi_base(); + } + else + { + base = _gic.dist_base; + } + + return base + gicv3_hwirq_convert_offset_index(hwirq, offset, index); +} + +static rt_bool_t gicv3_hwirq_peek(int hwirq, rt_uint32_t offset) +{ + rt_uint32_t index; + void *base = gicv3_hwirq_reg_base(hwirq, offset, &index); + + return !!HWREG32(base + (index / 32) * 4); +} + +static void gicv3_hwirq_poke(int hwirq, rt_uint32_t offset) +{ + rt_uint32_t index; + void *base = gicv3_hwirq_reg_base(hwirq, offset, &index); + + HWREG32(base + (index / 32) * 4) = 1 << (index % 32); +} + +static void gicv3_dist_init(void) +{ + rt_uint32_t i; + rt_uint64_t affinity; + void *base = _gic.dist_base; + rt_ubase_t mpidr = rt_cpu_mpidr_table[_init_cpu_id = rt_hw_cpu_id()]; + + _gic.line_nr = rt_min(GICD_TYPER_SPIS(_gic.gicd_typer), 1020U); + _gic.espi_nr = GICD_TYPER_ESPIS(_gic.gicd_typer); + + LOG_D("%d SPIs implemented", _gic.line_nr - 32); + LOG_D("%d Extended SPIs implemented", _gic.espi_nr); + + /* Disable the distributor */ + HWREG32(base + GICD_CTLR) = 0; + gicv3_dist_wait_for_rwp(); + + /* Non-secure Group-1 */ + for (i = 32; i < _gic.line_nr; i += 32) + { + HWREG32(base + GICD_IGROUPR + i / 8) = RT_UINT32_MAX; + } + + /* Disable, clear, group */ + for (i = 0; i < _gic.espi_nr; i += 4) + { + HWREG32(base + GICD_IPRIORITYRnE + i) = GICD_INT_DEF_PRI_X4; + + if (!(i % 16)) + { + HWREG32(base + GICD_ICFGRnE + i / 4) = 0; + + if (!(i % 32)) + { + HWREG32(base + GICD_ICENABLERnE + i / 8) = RT_UINT32_MAX; + HWREG32(base + GICD_ICACTIVERnE + i / 8) = RT_UINT32_MAX; + HWREG32(base + GICD_IGROUPRnE + i / 8) = RT_UINT32_MAX; + } + } + } + + gic_common_dist_config(base, _gic.line_nr, RT_NULL, RT_NULL); + + /* Enable the distributor */ + HWREG32(base + GICD_CTLR) = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; + gicv3_dist_wait_for_rwp(); + + affinity = ((rt_uint64_t)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | + MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | + MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | + MPIDR_AFFINITY_LEVEL(mpidr, 0)); + + /* Set all global interrupts to this CPU only. */ + for (i = 32; i < _gic.line_nr; ++i) + { + HWREG64(base + GICD_IROUTER + i * 8) = affinity; + } + + for (i = 0; i < _gic.espi_nr; ++i) + { + HWREG64(base + GICD_IROUTERnE + i * 8) = affinity; + } + + if (GICD_TYPER_NUM_LPIS(_gic.gicd_typer) > 1) + { + /* Max LPI = 8192 + Math.pow(2, num_LPIs + 1) - 1 */ + rt_size_t num_lpis = 1UL << (GICD_TYPER_NUM_LPIS(_gic.gicd_typer) + 1); + + _gic.lpi_nr = rt_min_t(int, num_lpis, 1UL << GICD_TYPER_ID_BITS(_gic.gicd_typer)); + } + else + { + _gic.lpi_nr = 1UL << GICD_TYPER_ID_BITS(_gic.gicd_typer); + } + + /* SPI + eSPI + LPIs */ + _gic.irq_nr = _gic.line_nr - 32 + _gic.espi_nr; + /* ITS will allocate the same number of lpi PIRQs */ + _gic.lpi_nr = rt_min_t(rt_size_t, (MAX_HANDLERS - _gic.irq_nr) / 2, _gic.lpi_nr); + _gic.irq_nr += _gic.lpi_nr; +} + +static void gicv3_redist_enable(rt_bool_t enable) +{ + void *base; + rt_uint32_t count = 1000000, waker; + + do { + if (_gic.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) + { + break; + } + + base = gicv3_percpu_redist_base(); + + waker = HWREG32(base + GICR_WAKER); + if (enable) + { + waker &= ~GICR_WAKER_ProcessorSleep; + } + else + { + waker |= GICR_WAKER_ProcessorSleep; + } + HWREG32(base + GICR_WAKER) = waker; + + if (!enable && !(HWREG32(base + GICR_WAKER) & GICR_WAKER_ProcessorSleep)) + { + break; + } + + while ((HWREG32(base + GICR_WAKER) & GICR_WAKER_ChildrenAsleep) != 0) + { + if (count-- == 0) + { + LOG_E("%s failed to %s", "Redistributor", enable ? "wakeup" : "sleep"); + break; + } + } + } while (0); +} + +static void gicv3_redist_init(void) +{ + void *base; + rt_uint32_t affinity; + int cpu_id = rt_hw_cpu_id(); + rt_bool_t find_ok = RT_TRUE; + rt_uint64_t mpidr = rt_cpu_mpidr_table[cpu_id], gicr_typer; + + affinity = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | + MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | + MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | + MPIDR_AFFINITY_LEVEL(mpidr, 0)); + + for (int i = 0; i < _gic.redist_regions_nr; ++i) + { + base = _gic.redist_regions[i].base; + + do { + gicr_typer = HWREG64(base + GICR_TYPER); + + if ((gicr_typer >> 32) == affinity) + { + rt_size_t ppi_nr = _gic.percpu_ppi_nr[cpu_id]; + rt_size_t typer_nr_ppis = GICR_TYPER_NR_PPIS(gicr_typer); + + _gic.percpu_ppi_nr[cpu_id] = rt_min(typer_nr_ppis, ppi_nr); + _gic.redist_percpu_base[cpu_id] = base; + + find_ok = RT_TRUE; + break; + } + + if (_gic.redist_stride) + { + base += _gic.redist_stride; + } + else + { + base += GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE; + + if (gicr_typer & GICR_TYPER_VLPIS) + { + base += GICR_VLPI_BASE_SIZE + GICR_RESERVED_SIZE; + } + } + } while (!(gicr_typer & GICR_TYPER_LAST)); + + if (find_ok) + { + break; + } + } + + if (find_ok) + { + gicv3_redist_enable(RT_TRUE); + } +} + +static void gicv3_cpu_init(void) +{ + void *base; + rt_size_t ppi_nr; + rt_uint64_t value; + int cpu_id = rt_hw_cpu_id(); +#ifdef ARCH_SUPPORT_HYP + _gicv3_eoi_mode_ns = RT_TRUE; +#else + _gicv3_eoi_mode_ns = !!rt_ofw_bootargs_select("pic.gicv3_eoimode", 0); +#endif + + base = gicv3_percpu_redist_sgi_base(); + ppi_nr = _gic.percpu_ppi_nr[cpu_id] + 16; + + for (rt_uint32_t i = 0; i < ppi_nr; i += 32) + { + HWREG32(base + GICR_IGROUPR0 + i / 8) = RT_UINT32_MAX; + } + + gic_common_cpu_config(base, ppi_nr, (void *)gicv3_redist_wait_for_rwp, &_gic.parent); + + read_gicreg(ICC_SRE_SYS, value); + value |= (1 << 0); + write_gicreg(ICC_SRE_SYS, value); + rt_hw_isb(); + + write_gicreg(ICC_PMR_SYS, 0xff); + + /* Enable group1 interrupt */ + write_gicreg(ICC_IGRPEN1_SYS, 1); + + write_gicreg(ICC_BPR1_SYS, 0); + + /* + * ICC_BPR0_EL1 determines the preemption group for both Group 0 and Group 1 + * interrupts. + * Targeted SGIs with affinity level 0 values of 0 - 255 are supported. + */ + value = ICC_CTLR_EL1_RSS | ICC_CTLR_EL1_CBPR_MASK; + if (_gicv3_eoi_mode_ns) + { + value |= ICC_CTLR_EL1_EOImode_drop; + } + write_gicreg(ICC_CTLR_SYS, value); +} + +static rt_err_t gicv3_irq_init(struct rt_pic *pic) +{ + gicv3_redist_init(); + gicv3_cpu_init(); + + return RT_EOK; +} + +static void gicv3_irq_ack(struct rt_pic_irq *pirq) +{ + if (!_gicv3_eoi_mode_ns) + { + write_gicreg(ICC_EOIR1_SYS, pirq->hwirq); + rt_hw_isb(); + } +} + +static void gicv3_irq_mask(struct rt_pic_irq *pirq) +{ + int hwirq = pirq->hwirq; + + gicv3_hwirq_poke(hwirq, GICD_ICENABLER); + + if (gicv3_hwirq_in_redist(hwirq)) + { + gicv3_redist_wait_for_rwp(); + } + else + { + gicv3_dist_wait_for_rwp(); + } +} + +static void gicv3_irq_unmask(struct rt_pic_irq *pirq) +{ + int hwirq = pirq->hwirq; + + gicv3_hwirq_poke(hwirq, GICD_ISENABLER); +} + +static void gicv3_irq_eoi(struct rt_pic_irq *pirq) +{ + if (_gicv3_eoi_mode_ns) + { + int hwirq = pirq->hwirq; + + if (hwirq < 8192) + { + write_gicreg(ICC_EOIR1_SYS, hwirq); + rt_hw_isb(); + + if (!_gicv3_arm64_2941627_erratum) + { + write_gicreg(ICC_DIR_SYS, hwirq); + rt_hw_isb(); + } + } + } +} + +static rt_err_t gicv3_irq_set_priority(struct rt_pic_irq *pirq, rt_uint32_t priority) +{ + void *base; + int hwirq = pirq->hwirq; + rt_uint32_t index, offset; + + if (gicv3_hwirq_in_redist(hwirq)) + { + base = gicv3_percpu_redist_sgi_base(); + } + else + { + base = _gic.dist_base; + } + + offset = gicv3_hwirq_convert_offset_index(hwirq, GICD_IPRIORITYR, &index); + HWREG8(base + offset + index) = priority; + + return RT_EOK; +} + +static rt_err_t gicv3_irq_set_affinity(struct rt_pic_irq *pirq, bitmap_t *affinity) +{ + rt_err_t ret = RT_EOK; + rt_uint64_t val; + rt_ubase_t mpidr; + rt_uint32_t offset, index; + int hwirq = pirq->hwirq, cpu_id = bitmap_next_set_bit(affinity, 0, RT_CPUS_NR); + + mpidr = rt_cpu_mpidr_table[cpu_id]; + + offset = gicv3_hwirq_convert_offset_index(hwirq, GICD_IROUTER, &index); + val = ((rt_uint64_t)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | + MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | + MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | + MPIDR_AFFINITY_LEVEL(mpidr, 0)); + + HWREG64(_gic.dist_base + offset + (index * 8)) = val; + + return ret; +} + +static rt_err_t gicv3_irq_set_triger_mode(struct rt_pic_irq *pirq, rt_uint32_t mode) +{ + void *base; + rt_err_t ret = RT_EOK; + int hwirq = pirq->hwirq; + rt_uint32_t index, offset; + + if (hwirq > 15) + { + if (gicv3_hwirq_in_redist(hwirq)) + { + base = gicv3_percpu_redist_sgi_base(); + } + else + { + base = _gic.dist_base; + } + + offset = gicv3_hwirq_convert_offset_index(hwirq, GICD_ICFGR, &index); + + ret = gic_common_configure_irq(base + offset, hwirq, mode, RT_NULL, RT_NULL); + } + else + { + ret = -RT_ENOSYS; + } + + return ret; +} + +static void gicv3_irq_send_ipi(struct rt_pic_irq *pirq, bitmap_t *cpumask) +{ +#define __mpidr_to_sgi_affinity(cluster_id, level) \ + (MPIDR_AFFINITY_LEVEL(cluster_id, level) << ICC_SGI1R_AFFINITY_##level##_SHIFT) + int cpu_id, last_cpu_id, limit; + rt_uint64_t initid, range_sel, target_list, cluster_id; + + range_sel = 0; + initid = ((pirq->hwirq) << ICC_SGI1R_SGI_ID_SHIFT); + + bitmap_for_each_set_bit(cpumask, cpu_id, RT_CPUS_NR) + { + rt_uint64_t mpidr = rt_cpu_mpidr_table[cpu_id]; + + cluster_id = mpidr & (~MPIDR_LEVEL_MASK); + target_list = 1 << ((mpidr & MPIDR_LEVEL_MASK) % ICC_SGI1R_TARGET_LIST_MAX); + limit = rt_min(cpu_id + ICC_SGI1R_TARGET_LIST_MAX, RT_CPUS_NR); + + last_cpu_id = cpu_id; + bitmap_for_each_set_bit_from(cpumask, cpu_id, cpu_id, limit) + { + rt_uint64_t mpidr = rt_cpu_mpidr_table[cpu_id]; + + if (cluster_id != (mpidr & (~MPIDR_LEVEL_MASK))) + { + range_sel = 0; + /* Don't break next cpuid */ + cpu_id = last_cpu_id; + break; + } + + last_cpu_id = cpu_id; + target_list |= 1 << ((mpidr & MPIDR_LEVEL_MASK) % ICC_SGI1R_TARGET_LIST_MAX); + } + + rt_hw_dsb(); + write_gicreg(ICC_SGI1R_SYS, + __mpidr_to_sgi_affinity(cluster_id, 3) | + (range_sel << ICC_SGI1R_RS_SHIFT) | + __mpidr_to_sgi_affinity(cluster_id, 2) | + initid | + __mpidr_to_sgi_affinity(cluster_id, 1) | + target_list); + rt_hw_isb(); + + ++range_sel; + } +#undef __mpidr_to_sgi_affinity +} + +static rt_err_t gicv3_irq_set_state(struct rt_pic *pic, int hwirq, int type, rt_bool_t state) +{ + rt_err_t err = RT_EOK; + rt_uint32_t offset = 0; + + if (hwirq >= 8192) + { + type = -1; + } + + switch (type) + { + case RT_IRQ_STATE_PENDING: + offset = state ? GICD_ISPENDR : GICD_ICPENDR; + break; + case RT_IRQ_STATE_ACTIVE: + offset = state ? GICD_ISACTIVER : GICD_ICACTIVER; + break; + case RT_IRQ_STATE_MASKED: + if (state) + { + struct rt_pic_irq pirq = {}; + + pirq.hwirq = hwirq; + gicv3_irq_mask(&pirq); + } + else + { + offset = GICD_ISENABLER; + } + break; + default: + err = -RT_EINVAL; + break; + } + + if (!err && offset) + { + gicv3_hwirq_poke(hwirq, offset); + } + + return err; +} + +static rt_err_t gicv3_irq_get_state(struct rt_pic *pic, int hwirq, int type, rt_bool_t *out_state) +{ + rt_err_t err = RT_EOK; + rt_uint32_t offset = 0; + + switch (type) + { + case RT_IRQ_STATE_PENDING: + offset = GICD_ISPENDR; + break; + case RT_IRQ_STATE_ACTIVE: + offset = GICD_ISACTIVER; + break; + case RT_IRQ_STATE_MASKED: + offset = GICD_ISENABLER; + break; + default: + err = -RT_EINVAL; + break; + } + + if (!err) + { + *out_state = gicv3_hwirq_peek(hwirq, offset); + } + + return err; +} + +static int gicv3_irq_map(struct rt_pic *pic, int hwirq, rt_uint32_t mode) +{ + struct rt_pic_irq *pirq; + int irq, hwirq_type, irq_index; + + hwirq_type = gicv3_hwirq_type(hwirq); + if (hwirq_type != LPI_TYPE) + { + irq_index = hwirq - GIC_SGI_NR; + } + else + { + irq_index = _gic.irq_nr - _gic.lpi_nr + hwirq - 8192; + } + pirq = rt_pic_find_irq(pic, irq_index); + + if (pirq && hwirq >= GIC_SGI_NR) + { + pirq->mode = mode; + pirq->priority = GICD_INT_DEF_PRI; + + switch (gicv3_hwirq_type(hwirq)) + { + case PPI_TYPE: + gic_fill_ppi_affinity(pirq->affinity); + break; + case SPI_TYPE: + case ESPI_TYPE: + RT_IRQ_AFFINITY_SET(pirq->affinity, _init_cpu_id); + default: + break; + } + + irq = rt_pic_config_irq(pic, irq_index, hwirq); + + if (irq >= 0 && mode != RT_IRQ_MODE_LEVEL_HIGH) + { + gicv3_irq_set_triger_mode(pirq, mode); + } + } + else + { + irq = -1; + } + + return irq; +} + +static rt_err_t gicv3_irq_parse(struct rt_pic *pic, struct rt_ofw_cell_args *args, struct rt_pic_irq *out_pirq) +{ + rt_err_t err = RT_EOK; + + if (args->args_count == 3) + { + out_pirq->mode = args->args[2] & RT_IRQ_MODE_MASK; + + switch (args->args[0]) + { + case 0: + /* SPI */ + out_pirq->hwirq = args->args[1] + 32; + break; + case 1: + /* PPI */ + out_pirq->hwirq = args->args[1] + 16; + break; + case 2: + /* ESPI */ + out_pirq->hwirq = args->args[1] + GIC_ESPI_BASE_INTID; + break; + case 3: + /* EPPI */ + out_pirq->hwirq = args->args[1] + GIC_EPPI_BASE_INTID; + break; + case GIC_IRQ_TYPE_LPI: + /* LPI */ + out_pirq->hwirq = args->args[1]; + break; + case GIC_IRQ_TYPE_PARTITION: + + out_pirq->hwirq = args->args[1]; + + if (args->args[1] >= 16) + { + out_pirq->hwirq += GIC_EPPI_BASE_INTID - 16; + } + else + { + out_pirq->hwirq += 16; + } + break; + default: + err = -RT_ENOSYS; + break; + } + } + else + { + err = -RT_EINVAL; + } + + return err; +} + +const static struct rt_pic_ops gicv3_ops = +{ + .name = "GICv3", + .irq_init = gicv3_irq_init, + .irq_ack = gicv3_irq_ack, + .irq_mask = gicv3_irq_mask, + .irq_unmask = gicv3_irq_unmask, + .irq_eoi = gicv3_irq_eoi, + .irq_set_priority = gicv3_irq_set_priority, + .irq_set_affinity = gicv3_irq_set_affinity, + .irq_set_triger_mode = gicv3_irq_set_triger_mode, + .irq_send_ipi = gicv3_irq_send_ipi, + .irq_set_state = gicv3_irq_set_state, + .irq_get_state = gicv3_irq_get_state, + .irq_map = gicv3_irq_map, + .irq_parse = gicv3_irq_parse, +}; + +static rt_bool_t gicv3_handler(void *data) +{ + rt_bool_t res = RT_FALSE; + int hwirq; + struct gicv3 *gic = data; + + read_gicreg(ICC_IAR1_SYS, hwirq); + + if (!(hwirq >= 1020 && hwirq <= 1023)) + { + struct rt_pic_irq *pirq; + + if (hwirq < GIC_SGI_NR) + { + rt_hw_rmb(); + + pirq = rt_pic_find_ipi(&gic->parent, hwirq); + } + else + { + int irq_index; + + if (hwirq < 8192) + { + irq_index = hwirq - GIC_SGI_NR; + } + else + { + irq_index = gic->irq_nr - gic->lpi_nr + hwirq - 8192; + } + + pirq = rt_pic_find_irq(&gic->parent, irq_index); + } + + gicv3_irq_ack(pirq); + + rt_pic_handle_isr(pirq); + + gicv3_irq_eoi(pirq); + + res = RT_TRUE; + } + + return res; +} + +static rt_err_t gicv3_enable_quirk_msm8996(void *data) +{ + struct gicv3 *gic = data; + + gic->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; + + return RT_EOK; +} + +static rt_err_t gicv3_enable_quirk_arm64_2941627(void *data) +{ + _gicv3_arm64_2941627_erratum = RT_TRUE; + return RT_EOK; +} + +static const struct gic_quirk _gicv3_quirks[] = +{ + { + .desc = "GICv3: Qualcomm MSM8996 broken firmware", + .compatible = "qcom,msm8996-gic-v3", + .init = gicv3_enable_quirk_msm8996, + }, + { + /* GIC-700: 2941627 workaround - IP variant [0,1] */ + .desc = "GICv3: ARM64 erratum 2941627", + .iidr = 0x0400043b, + .iidr_mask = 0xff0e0fff, + .init = gicv3_enable_quirk_arm64_2941627, + }, + { + /* GIC-700: 2941627 workaround - IP variant [2] */ + .desc = "GICv3: ARM64 erratum 2941627", + .iidr = 0x0402043b, + .iidr_mask = 0xff0f0fff, + .init = gicv3_enable_quirk_arm64_2941627, + }, + { /* sentinel */ } +}; + +static rt_err_t gicv3_iomap_init(rt_uint64_t *regs) +{ + rt_err_t ret = RT_EOK; + int idx; + char *name; + + do { + /* GICD->GICR */ + _gic.dist_size = regs[1]; + _gic.dist_base = rt_ioremap((void *)regs[0], _gic.dist_size); + if (!_gic.dist_base) + { + name = "Distributor"; + idx = 0; + ret = -RT_ERROR; + break; + } + + name = "Redistributor"; + + _gic.redist_regions = rt_malloc(sizeof(_gic.redist_regions[0]) * _gic.redist_regions_nr); + if (!_gic.redist_regions) + { + idx = -1; + ret = -RT_ENOMEM; + LOG_E("No memory to save %s", name); + break; + } + + for (int i = 0, off = 2; i < _gic.redist_regions_nr; ++i) + { + void *base = (void *)regs[off++]; + rt_size_t size = regs[off++]; + _gic.redist_regions[i].size = size; + _gic.redist_regions[i].base = rt_ioremap(base, size); + _gic.redist_regions[i].base_phy = base; + + if (!base) + { + idx = 1; + ret = -RT_ERROR; + break; + } + } + + if (ret) + { + break; + } + + /* ArchRev[4:7] */ + _gic.version = HWREG32(_gic.dist_base + GICD_PIDR2) >> 4; + } while (0); + + if (ret && idx >= 0) + { + RT_UNUSED(name); + LOG_E("%s IO[%p, %p] map fail", name[idx], regs[idx * 2], regs[idx * 2 + 1]); + } + + return ret; +} + +static void gicv3_init(void) +{ +#define __dist_espi_regs_do(func, expr, ...) \ + __VA_ARGS__(*func(GICD_IGROUPR) expr GICD_IGROUPRnE); \ + __VA_ARGS__(*func(GICD_ISENABLER) expr GICD_ISENABLERnE); \ + __VA_ARGS__(*func(GICD_ICENABLER) expr GICD_ICENABLERnE); \ + __VA_ARGS__(*func(GICD_ISPENDR) expr GICD_ISPENDRnE); \ + __VA_ARGS__(*func(GICD_ICPENDR) expr GICD_ICPENDRnE); \ + __VA_ARGS__(*func(GICD_ISACTIVER) expr GICD_ISACTIVERnE); \ + __VA_ARGS__(*func(GICD_ICACTIVER) expr GICD_ICACTIVERnE); \ + __VA_ARGS__(*func(GICD_IPRIORITYR) expr GICD_IPRIORITYRnE); \ + __VA_ARGS__(*func(GICD_ICFGR) expr GICD_ICFGRnE); \ + __VA_ARGS__(*func(GICD_IROUTER) expr GICD_IROUTERnE); + + /* Map registers for ESPI */ + __dist_espi_regs_do(gicv3_dist_espi_reg, =); + __dist_espi_regs_do(gicv3_dist_espi_reg, ==, RT_ASSERT); +#undef __dist_espi_regs_do + + _gic.gicd_typer = HWREG32(_gic.dist_base + GICD_TYPER); + + gic_common_init_quirk_hw(HWREG32(_gic.dist_base + GICD_IIDR), _gicv3_quirks, &_gic.parent); + gicv3_dist_init(); + + _gic.parent.priv_data = &_gic; + _gic.parent.ops = &gicv3_ops; + + rt_pic_linear_irq(&_gic.parent, _gic.irq_nr - GIC_SGI_NR); + gic_common_sgi_config(_gic.dist_base, &_gic.parent, 0); + + rt_pic_add_traps(gicv3_handler, &_gic); + + rt_pic_user_extends(&_gic.parent); +} + +static void gicv3_init_fail(void) +{ + if (_gic.dist_base) + { + rt_iounmap(_gic.dist_base); + } + if (_gic.redist_regions) + { + for (int i = 0; i < _gic.redist_regions_nr; ++i) + { + if (_gic.redist_regions[i].base) + { + rt_iounmap(_gic.redist_regions[i].base); + } + } + + rt_free(_gic.redist_regions); + } + + rt_memset(&_gic, 0, sizeof(_gic)); +} + +static rt_err_t gicv3_ofw_init(struct rt_ofw_node *np, const struct rt_ofw_node_id *id) +{ + rt_err_t err = RT_EOK; + + do { + rt_size_t reg_nr_max; + rt_err_t msi_init = -RT_ENOSYS; + rt_uint32_t redist_regions_nr; + rt_uint64_t *regs, redist_stride; + + if (rt_ofw_prop_read_u32(np, "#redistributor-regions", &redist_regions_nr)) + { + redist_regions_nr = 1; + } + + /* GICD + n * GICR */ + reg_nr_max = 2 + (2 * redist_regions_nr); + regs = rt_calloc(1, sizeof(rt_uint64_t) * reg_nr_max); + + if (!regs) + { + err = -RT_ENOMEM; + break; + } + + rt_ofw_get_address_array(np, reg_nr_max, regs); + _gic.redist_regions_nr = redist_regions_nr; + + err = gicv3_iomap_init(regs); + rt_free(regs); + + if (err) + { + break; + } + + if (_gic.version != 3 && _gic.version != 4) + { + LOG_E("Version = %d is not support", _gic.version); + err = -RT_EINVAL; + break; + } + + if (rt_ofw_prop_read_u64(np, "redistributor-stride", &redist_stride)) + { + redist_stride = 0; + } + _gic.redist_stride = redist_stride; + + gic_common_init_quirk_ofw(np, _gicv3_quirks, &_gic.parent); + gicv3_init(); + + rt_ofw_data(np) = &_gic.parent; + + #ifdef RT_PIC_ARM_GIC_V3_ITS + msi_init = gicv3_its_ofw_probe(np, id); + #endif + + /* V2M or ITS only */ + if (msi_init) + { + #ifdef RT_PIC_ARM_GIC_V2M + gicv2m_ofw_probe(np, id); + #endif + } + } while (0); + + if (err) + { + gicv3_init_fail(); + } + + return err; +} + +static const struct rt_ofw_node_id gicv3_ofw_ids[] = +{ + { .compatible = "arm,gic-v3" }, + { /* sentinel */ } +}; +RT_PIC_OFW_DECLARE(gicv3, gicv3_ofw_ids, gicv3_ofw_init); diff --git a/components/drivers/pic/pic-gicv3.h b/components/drivers/pic/pic-gicv3.h new file mode 100644 index 000000000000..18abb0d7da36 --- /dev/null +++ b/components/drivers/pic/pic-gicv3.h @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2013-07-20 Bernard first version + * 2014-04-03 Grissiom many enhancements + * 2018-11-22 Jesven add rt_hw_ipi_send() + * add rt_hw_ipi_handler_install() + * 2023-02-01 GuEe-GUI move macros to header + */ + +#ifndef __IRQ_GICV3_H__ +#define __IRQ_GICV3_H__ + +#include + +#include + +#include +#include +#include + +/* Distributor registers */ +#define GICD_CTLR 0x0000 +#define GICD_TYPER 0x0004 +#define GICD_IIDR 0x0008 +#define GICD_TYPER2 0x000C +#define GICD_STATUSR 0x0010 +#define GICD_SETSPI_NSR 0x0040 +#define GICD_CLRSPI_NSR 0x0048 +#define GICD_SETSPI_SR 0x0050 +#define GICD_CLRSPI_SR 0x0058 +#define GICD_IGROUPR 0x0080 +#define GICD_ISENABLER 0x0100 +#define GICD_ICENABLER 0x0180 +#define GICD_ISPENDR 0x0200 +#define GICD_ICPENDR 0x0280 +#define GICD_ISACTIVER 0x0300 +#define GICD_ICACTIVER 0x0380 +#define GICD_IPRIORITYR 0x0400 +#define GICD_ICFGR 0x0C00 +#define GICD_IGRPMODR 0x0D00 +#define GICD_NSACR 0x0E00 +#define GICD_IGROUPRnE 0x1000 +#define GICD_ISENABLERnE 0x1200 +#define GICD_ICENABLERnE 0x1400 +#define GICD_ISPENDRnE 0x1600 +#define GICD_ICPENDRnE 0x1800 +#define GICD_ISACTIVERnE 0x1A00 +#define GICD_ICACTIVERnE 0x1C00 +#define GICD_IPRIORITYRnE 0x2000 +#define GICD_ICFGRnE 0x3000 +#define GICD_IROUTER 0x6000 +#define GICD_IROUTERnE 0x8000 +#define GICD_IDREGS 0xFFD0 +#define GICD_PIDR2 0xFFE8 + +#define GICD_ITARGETSR 0x0800 +#define GICD_SGIR 0x0F00 +#define GICD_CPENDSGIR 0x0F10 +#define GICD_SPENDSGIR 0x0F20 + +#define GICD_CTLR_RWP (1U << 31) +#define GICD_CTLR_nASSGIreq (1U << 8) +#define GICD_CTLR_DS (1U << 6) +#define GICD_CTLR_ARE_NS (1U << 4) +#define GICD_CTLR_ENABLE_G1A (1U << 1) +#define GICD_CTLR_ENABLE_G1 (1U << 0) + +#define GICD_TYPER_RSS (1U << 26) +#define GICD_TYPER_LPIS (1U << 17) +#define GICD_TYPER_MBIS (1U << 16) +#define GICD_TYPER_ESPI (1U << 8) +#define GICD_TYPER_ID_BITS(t) ((((t) >> 19) & 0x1f) + 1) +#define GICD_TYPER_NUM_LPIS(t) ((((t) >> 11) & 0x1f) + 1) +#define GICD_TYPER_SPIS(t) ((((t) & 0x1f) + 1) * 32) +#define GICD_TYPER_ESPIS(t) (((t) & GICD_TYPER_ESPI) ? GICD_TYPER_SPIS((t) >> 27) : 0) + +/* Redistributor registers */ +#define GICR_CTLR 0x0000 +#define GICR_IIDR 0x0004 +#define GICR_TYPER 0x0008 +#define GICR_STATUSR 0x0010 +#define GICR_WAKER 0x0014 +#define GICR_MPAMIDR 0x0018 +#define GICR_PARTIDR 0x001C +#define GICR_SETLPIR 0x0040 +#define GICR_CLRLPIR 0x0048 +#define GICR_PROPBASER 0x0070 +#define GICR_PENDBASER 0x0078 +#define GICR_INVLPIR 0x00A0 +#define GICR_INVALLR 0x00B0 +#define GICR_SYNCR 0x00C0 +#define GICR_PIDR2 GICD_PIDR2 + +#define GICR_CTLR_ENABLE_LPIS (1UL << 0) +#define GICR_CTLR_CES (1UL << 1) +#define GICR_CTLR_IR (1UL << 2) +#define GICR_CTLR_RWP (1UL << 3) + +#define GICR_TYPER_CPU_NO(r) (((r) >> 8) & 0xffff) + +#define GICR_RD_BASE_SIZE (64 * SIZE_KB) +#define GICR_SGI_OFFSET (64 * SIZE_KB) +#define GICR_SGI_BASE_SIZE GICR_SGI_OFFSET + +/* Re-Distributor registers, offsets from SGI_base */ +#define GICR_IGROUPR0 GICD_IGROUPR +#define GICR_ISENABLER0 GICD_ISENABLER +#define GICR_ICENABLER0 GICD_ICENABLER +#define GICR_ISPENDR0 GICD_ISPENDR +#define GICR_ICPENDR0 GICD_ICPENDR +#define GICR_ISACTIVER0 GICD_ISACTIVER +#define GICR_ICACTIVER0 GICD_ICACTIVER +#define GICR_IPRIORITYR0 GICD_IPRIORITYR +#define GICR_ICFGR0 GICD_ICFGR +#define GICR_IGRPMODR0 GICD_IGRPMODR +#define GICR_NSACR GICD_NSACR + +#define GICR_TYPER_PLPIS (1U << 0) +#define GICR_TYPER_VLPIS (1U << 1) +#define GICR_TYPER_DIRTY (1U << 2) +#define GICR_TYPER_DirectLPIS (1U << 3) +#define GICR_TYPER_LAST (1U << 4) +#define GICR_TYPER_RVPEID (1U << 7) +#define GICR_TYPER_COM_LPI_AFF RT_GENMASK_ULL(25, 24) +#define GICR_TYPER_AFFINITY RT_GENMASK_ULL(63, 32) + +#define GICR_INVLPIR_INTID RT_GENMASK_ULL(31, 0) +#define GICR_INVLPIR_VPEID RT_GENMASK_ULL(47, 32) +#define GICR_INVLPIR_V RT_GENMASK_ULL(63, 63) + +#define GICR_INVALLR_VPEID GICR_INVLPIR_VPEID +#define GICR_INVALLR_V GICR_INVLPIR_V + +#define GICR_VLPI_BASE_SIZE (64 * SIZE_KB) +#define GICR_RESERVED_SIZE (64 * SIZE_KB) + +#define GIC_V3_REDIST_SIZE 0x20000 + +#define GICR_TYPER_NR_PPIS(t) (16 + ({ int __ppinum = (((t) >> 27) & 0x1f); __ppinum <= 2 ? __ppinum : 0; }) * 32) + +#define GICR_WAKER_ProcessorSleep (1U << 1) +#define GICR_WAKER_ChildrenAsleep (1U << 2) + +#define GICR_PROPBASER_IDBITS_MASK (0x1f) +#define GICR_PROPBASER_ADDRESS(x) ((x) & RT_GENMASK_ULL(51, 12)) +#define GICR_PENDBASER_ADDRESS(x) ((x) & RT_GENMASK_ULL(51, 16)) + +/* ITS registers */ +#define GITS_CTLR 0x0000 +#define GITS_IIDR 0x0004 +#define GITS_TYPER 0x0008 +#define GITS_MPIDR 0x0018 +#define GITS_CBASER 0x0080 +#define GITS_CWRITER 0x0088 +#define GITS_CREADR 0x0090 +#define GITS_BASER 0x0100 +#define GITS_IDREGS_BASE 0xffd0 +#define GITS_PIDR0 0xffe0 +#define GITS_PIDR1 0xffe4 +#define GITS_PIDR2 GICR_PIDR2 +#define GITS_PIDR4 0xffd0 +#define GITS_CIDR0 0xfff0 +#define GITS_CIDR1 0xfff4 +#define GITS_CIDR2 0xfff8 +#define GITS_CIDR3 0xfffc + +#define GITS_TRANSLATER 0x10040 + +#define GITS_SGIR 0x20020 + +#define GITS_SGIR_VPEID RT_GENMASK_ULL(47, 32) +#define GITS_SGIR_VINTID RT_GENMASK_ULL(3, 0) + +#define GITS_CTLR_ENABLE (1U << 0) +#define GITS_CTLR_ImDe (1U << 1) +#define GITS_CTLR_ITS_NUMBER_SHIFT 4 +#define GITS_CTLR_ITS_NUMBER (0xFU << GITS_CTLR_ITS_NUMBER_SHIFT) +#define GITS_CTLR_QUIESCENT (1U << 31) + +#define GITS_TYPER_PLPIS (1UL << 0) +#define GITS_TYPER_VLPIS (1UL << 1) +#define GITS_TYPER_ITT_ENTRY_SIZE_SHIFT 4 +#define GITS_TYPER_ITT_ENTRY_SIZE RT_GENMASK_ULL(7, 4) +#define GITS_TYPER_IDBITS_SHIFT 8 +#define GITS_TYPER_DEVBITS_SHIFT 13 +#define GITS_TYPER_DEVBITS RT_GENMASK_ULL(17, 13) +#define GITS_TYPER_PTA (1UL << 19) +#define GITS_TYPER_HCC_SHIFT 24 +#define GITS_TYPER_HCC(r) (((r) >> GITS_TYPER_HCC_SHIFT) & 0xff) +#define GITS_TYPER_VMOVP (1ULL << 37) +#define GITS_TYPER_VMAPP (1ULL << 40) +#define GITS_TYPER_SVPET RT_GENMASK_ULL(42, 41) + +/* + * ITS commands + */ +#define GITS_CMD_MAPD 0x08 +#define GITS_CMD_MAPC 0x09 +#define GITS_CMD_MAPTI 0x0a +#define GITS_CMD_MAPI 0x0b +#define GITS_CMD_MOVI 0x01 +#define GITS_CMD_DISCARD 0x0f +#define GITS_CMD_INV 0x0c +#define GITS_CMD_MOVALL 0x0e +#define GITS_CMD_INVALL 0x0d +#define GITS_CMD_INT 0x03 +#define GITS_CMD_CLEAR 0x04 +#define GITS_CMD_SYNC 0x05 + +/* ITS Config Area */ +#define GITS_LPI_CFG_GROUP1 (1 << 1) +#define GITS_LPI_CFG_ENABLED (1 << 0) + +/* ITS Command Queue Descriptor */ +#define GITS_BASER_SHAREABILITY_SHIFT 10 +#define GITS_BASER_INNER_CACHEABILITY_SHIFT 59 +#define GITS_BASER_OUTER_CACHEABILITY_SHIFT 53 +#define GITS_CBASER_VALID (1UL << 63) +#define GITS_CBASER_SHAREABILITY_SHIFT 10 +#define GITS_CBASER_INNER_CACHEABILITY_SHIFT 59 +#define GITS_CBASER_OUTER_CACHEABILITY_SHIFT 53 +#define GICR_PBASER_SHAREABILITY_SHIFT 10 +#define GICR_PBASER_INNER_CACHEABILITY_SHIFT 7 +#define GICR_PBASER_OUTER_CACHEABILITY_SHIFT 56 + +#define GITS_BASER_NR_REGS 8 +#define GITS_BASERn(idx) (GITS_BASER + sizeof(rt_uint64_t) * idx) + +#define GITS_BASER_VALID (1ULL << 63) +#define GITS_BASER_INDIRECT (1ULL << 62) +#define GITS_BASER_TYPE_SHIFT 56 +#define GITS_BASER_TYPE(r) (((r) >> GITS_BASER_TYPE_SHIFT) & 7) +#define GITS_BASER_ENTRY_SIZE_SHIFT 48 +#define GITS_BASER_ENTRY_SIZE(r) ((((r) >> GITS_BASER_ENTRY_SIZE_SHIFT) & 0x1f) + 1) +#define GITS_BASER_PAGE_SIZE_SHIFT 8 +#define GITS_BASER_PAGE_SIZE_4K (0ULL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_16K (1ULL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_64K (2ULL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_MASK (3ULL << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGES_SHIFT 0 + +#define GITS_LVL1_ENTRY_SIZE 8UL + +#define GITS_BASER_TYPE_NONE 0 +#define GITS_BASER_TYPE_DEVICE 1 +#define GITS_BASER_TYPE_VPE 2 +#define GITS_BASER_TYPE_RESERVED3 3 +#define GITS_BASER_TYPE_COLLECTION 4 +#define GITS_BASER_TYPE_RESERVED5 5 +#define GITS_BASER_TYPE_RESERVED6 6 +#define GITS_BASER_TYPE_RESERVED7 7 + +#define GITS_BASER_CACHEABILITY(reg, inner_outer, type) \ + (GIC_BASER_CACHE_##type << reg##_##inner_outer##_CACHEABILITY_SHIFT) +#define GITS_BASER_SHAREABILITY(reg, type) \ + (GIC_BASER_##type << reg##_SHAREABILITY_SHIFT) + +#define GIC_BASER_CACHE_DnGnRnE 0x0UL /* Device-nGnRnE. */ +#define GIC_BASER_CACHE_NIN 0x1UL /* Normal Inner Non-cacheable. */ +#define GIC_BASER_CACHE_NIRAWT 0x2UL /* Normal Inner Cacheable Read-allocate, Write-through. */ +#define GIC_BASER_CACHE_NIRAWB 0x3UL /* Normal Inner Cacheable Read-allocate, Write-back. */ +#define GIC_BASER_CACHE_NIWAWT 0x4UL /* Normal Inner Cacheable Write-allocate, Write-through. */ +#define GIC_BASER_CACHE_NIWAWB 0x5UL /* Normal Inner Cacheable Write-allocate, Write-back. */ +#define GIC_BASER_CACHE_NIRAWAWT 0x6UL /* Normal Inner Cacheable Read-allocate, Write-allocate, Write-through. */ +#define GIC_BASER_CACHE_NIRAWAWB 0x7UL /* Normal Inner Cacheable Read-allocate, Write-allocate, Write-back. */ +#define GIC_BASER_CACHE_MASK 0x7UL +#define GIC_BASER_SHARE_NS 0x0UL /* Non-shareable. */ +#define GIC_BASER_SHARE_IS 0x1UL /* Inner Shareable. */ +#define GIC_BASER_SHARE_OS 0x2UL /* Outer Shareable. */ +#define GIC_BASER_SHARE_RES 0x3UL /* Reserved. Treated as 0b00 */ +#define GIC_BASER_SHARE_MASK 0x3UL + +#define GITS_BASER_InnerShareable GITS_BASER_SHAREABILITY(GITS_BASER, SHARE_IS) +#define GITS_BASER_SHARE_MASK_ALL GITS_BASER_SHAREABILITY(GITS_BASER, SHARE_MASK) +#define GITS_BASER_INNER_MASK_ALL GITS_BASER_CACHEABILITY(GITS_BASER, INNER, MASK) +#define GITS_BASER_nCnB GITS_BASER_CACHEABILITY(GITS_BASER, INNER, DnGnRnE) +#define GITS_BASER_nC GITS_BASER_CACHEABILITY(GITS_BASER, INNER, NIN) +#define GITS_BASER_RaWt GITS_BASER_CACHEABILITY(GITS_BASER, INNER, NIRAWT) +#define GITS_BASER_RaWb GITS_BASER_CACHEABILITY(GITS_BASER, INNER, NIRAWB) +#define GITS_BASER_WaWt GITS_BASER_CACHEABILITY(GITS_BASER, INNER, NIWAWT) +#define GITS_BASER_WaWb GITS_BASER_CACHEABILITY(GITS_BASER, INNER, NIWAWB) +#define GITS_BASER_RaWaWt GITS_BASER_CACHEABILITY(GITS_BASER, INNER, NIRAWAWT) +#define GITS_BASER_RaWaWb GITS_BASER_CACHEABILITY(GITS_BASER, INNER, NIRAWAWB) + +#define GITS_CBASER_InnerShareable GITS_BASER_SHAREABILITY(GITS_CBASER, SHARE_IS) +#define GITS_CBASER_SHARE_MASK_ALL GITS_BASER_SHAREABILITY(GITS_CBASER, SHARE_MASK) +#define GITS_CBASER_INNER_MASK_ALL GITS_BASER_CACHEABILITY(GITS_CBASER, INNER, MASK) +#define GITS_CBASER_nCnB GITS_BASER_CACHEABILITY(GITS_CBASER, INNER, DnGnRnE) +#define GITS_CBASER_nC GITS_BASER_CACHEABILITY(GITS_CBASER, INNER, NIN) +#define GITS_CBASER_RaWt GITS_BASER_CACHEABILITY(GITS_CBASER, INNER, NIRAWT) +#define GITS_CBASER_RaWb GITS_BASER_CACHEABILITY(GITS_CBASER, INNER, NIRAWB) +#define GITS_CBASER_WaWt GITS_BASER_CACHEABILITY(GITS_CBASER, INNER, NIWAWT) +#define GITS_CBASER_WaWb GITS_BASER_CACHEABILITY(GITS_CBASER, INNER, NIWAWB) +#define GITS_CBASER_RaWaWt GITS_BASER_CACHEABILITY(GITS_CBASER, INNER, NIRAWAWT) +#define GITS_CBASER_RaWaWb GITS_BASER_CACHEABILITY(GITS_CBASER, INNER, NIRAWAWB) + +#define GICR_PBASER_InnerShareable GITS_BASER_SHAREABILITY(GICR_PBASER, SHARE_IS) +#define GICR_PBASER_SHARE_MASK_ALL GITS_BASER_SHAREABILITY(GICR_PBASER, SHARE_MASK) +#define GICR_PBASER_INNER_MASK_ALL GITS_BASER_CACHEABILITY(GICR_PBASER, INNER, MASK) +#define GICR_PBASER_nCnB GITS_BASER_CACHEABILITY(GICR_PBASER, INNER, DnGnRnE) +#define GICR_PBASER_nC GITS_BASER_CACHEABILITY(GICR_PBASER, INNER, NIN) +#define GICR_PBASER_RaWt GITS_BASER_CACHEABILITY(GICR_PBASER, INNER, NIRAWT) +#define GICR_PBASER_RaWb GITS_BASER_CACHEABILITY(GICR_PBASER, INNER, NIRAWB) +#define GICR_PBASER_WaWt GITS_BASER_CACHEABILITY(GICR_PBASER, INNER, NIWAWT) +#define GICR_PBASER_WaWb GITS_BASER_CACHEABILITY(GICR_PBASER, INNER, NIWAWB) +#define GICR_PBASER_RaWaWt GITS_BASER_CACHEABILITY(GICR_PBASER, INNER, NIRAWAWT) +#define GICR_PBASER_RaWaWb GITS_BASER_CACHEABILITY(GICR_PBASER, INNER, NIRAWAWB) + +#define GIC_EPPI_BASE_INTID 1056 +#define GIC_ESPI_BASE_INTID 4096 + +#define GIC_IRQ_TYPE_LPI 0xa110c8ed +#define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) + +#define read_gicreg(reg, out) sysreg_read(reg, out) +#define write_gicreg(reg, in) sysreg_write(reg, in) + +#define ICC_CTLR_EOImode 0x2 +#define ICC_PMR_MASK 0xff +#define ICC_PMR_DEFAULT 0xf0 +#define ICC_IGRPEN1_EN 0x1 + +#define ICC_SGIR_AFF3_SHIFT 48 +#define ICC_SGIR_AFF2_SHIFT 32 +#define ICC_SGIR_AFF1_SHIFT 16 +#define ICC_SGIR_TARGET_MASK 0xffff +#define ICC_SGIR_IRQN_SHIFT 24 +#define ICC_SGIR_ROUTING_BIT (1ULL << 40) + +#define ICC_SGI1R_TARGET_LIST_SHIFT 0 +#define ICC_SGI1R_TARGET_LIST_MASK (0xffff << ICC_SGI1R_TARGET_LIST_SHIFT) +#define ICC_SGI1R_TARGET_LIST_MAX 16 +#define ICC_SGI1R_AFFINITY_1_SHIFT 16 +#define ICC_SGI1R_AFFINITY_1_MASK (0xff << ICC_SGI1R_AFFINITY_1_SHIFT) +#define ICC_SGI1R_SGI_ID_SHIFT 24 +#define ICC_SGI1R_SGI_ID_MASK (0xfULL << ICC_SGI1R_SGI_ID_SHIFT) +#define ICC_SGI1R_AFFINITY_2_SHIFT 32 +#define ICC_SGI1R_AFFINITY_2_MASK (0xffULL << ICC_SGI1R_AFFINITY_2_SHIFT) +#define ICC_SGI1R_IRQ_ROUTING_MODE_BIT 40 +#define ICC_SGI1R_RS_SHIFT 44 +#define ICC_SGI1R_RS_MASK (0xfULL << ICC_SGI1R_RS_SHIFT) +#define ICC_SGI1R_AFFINITY_3_SHIFT 48 +#define ICC_SGI1R_AFFINITY_3_MASK (0xffULL << ICC_SGI1R_AFFINITY_3_SHIFT) + +#define ICC_CTLR_EL1_CBPR_SHIFT 0 +#define ICC_CTLR_EL1_CBPR_MASK (1 << ICC_CTLR_EL1_CBPR_SHIFT) +#define ICC_CTLR_EL1_EOImode_SHIFT (1) +#define ICC_CTLR_EL1_EOImode_drop (1U << ICC_CTLR_EL1_EOImode_SHIFT) +#define ICC_CTLR_EL1_EOImode_drop_dir (0U << ICC_CTLR_EL1_EOImode_SHIFT) +#define ICC_CTLR_EL1_PRI_BITS_SHIFT (8) +#define ICC_CTLR_EL1_PRI_BITS_MASK (0x7 << ICC_CTLR_EL1_PRI_BITS_SHIFT) +#define ICC_CTLR_EL1_RSS (0x1 << 18) +#define ICC_CTLR_EL1_ExtRange (0x1 << 19) + +struct gicv3 +{ + struct rt_pic parent; + + int version; + int irq_nr; + rt_uint32_t gicd_typer; + rt_size_t line_nr; + rt_size_t espi_nr; + rt_size_t lpi_nr; + rt_ubase_t flags; + + void *dist_base; + rt_size_t dist_size; + + void *redist_percpu_base[RT_CPUS_NR]; + rt_uint64_t redist_percpu_flags[RT_CPUS_NR]; + rt_size_t percpu_ppi_nr[RT_CPUS_NR]; + + struct + { + void *base; + void *base_phy; + rt_size_t size; + } *redist_regions; + rt_uint64_t redist_flags; + rt_size_t redist_stride; + rt_size_t redist_regions_nr; +}; + +#endif /* __IRQ_GICV3_H__ */