/* * Copyright (c) 2006-2022, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * * Change Logs: * Date Author Notes * 2022-11-26 GuEe-GUI first version */ #include #include #define DBG_TAG "pcie.brcmstb" #define DBG_LVL DBG_INFO #include #include #include #include #include /* BRCM_PCIE_CAP_REGS - Offset for the mandatory capability config regs */ #define BRCM_PCIE_CAP_REGS 0x00ac /* Broadcom STB PCIe Register Offsets */ #define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1 0x0188 #define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK 0xc #define PCIE_RC_CFG_VENDOR_SPECIFIC_REG1_LITTLE_ENDIAN 0x0 #define PCIE_RC_CFG_PRIV1_ID_VAL3 0x043c #define PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK 0xffffff #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY 0x04dc #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_MAX_LINK_WIDTH_MASK 0x1f0 #define PCIE_RC_CFG_PRIV1_ROOT_CAP 0x4f8 #define PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK 0xf8 #define PCIE_RC_TL_VDM_CTL0 0x0a20 #define PCIE_RC_TL_VDM_CTL0_VDM_ENABLED_MASK 0x10000 #define PCIE_RC_TL_VDM_CTL0_VDM_IGNORETAG_MASK 0x20000 #define PCIE_RC_TL_VDM_CTL0_VDM_IGNOREVNDRID_MASK 0x40000 #define PCIE_RC_TL_VDM_CTL1 0x0a0c #define PCIE_RC_TL_VDM_CTL1_VDM_VNDRID0_MASK 0x0000ffff #define PCIE_RC_TL_VDM_CTL1_VDM_VNDRID1_MASK 0xffff0000 #define PCIE_RC_DL_MDIO_ADDR 0x1100 #define PCIE_RC_DL_MDIO_WR_DATA 0x1104 #define PCIE_RC_DL_MDIO_RD_DATA 0x1108 #define PCIE_RC_PL_REG_PHY_CTL_1 0x1804 #define PCIE_RC_PL_REG_PHY_CTL_1_REG_P2_POWERDOWN_ENA_NOSYNC_MASK 0x8 #define PCIE_RC_PL_PHY_CTL_15 0x184c #define PCIE_RC_PL_PHY_CTL_15_DIS_PLL_PD_MASK 0x400000 #define PCIE_RC_PL_PHY_CTL_15_PM_CLK_PERIOD_MASK 0xff #define PCIE_RC_PL_STATS_CTRL 0x1940 #define PCIE_RC_PL_STATS_CTRL_EN_MASK 0x1 #define PCIE_RC_PL_STATS_CTRL_LEN_MASK 0xfffffff0 #define PCIE_RC_PL_STATS_TXTLP_LO 0x1944 #define PCIE_RC_PL_STATS_TXTLP_HI 0x1948 #define PCIE_RC_PL_STATS_TXDLLP_LO 0x194c #define PCIE_RC_PL_STATS_TXDLLP_HI 0x1950 #define PCIE_RC_PL_STATS_RXTLP_LO 0x195c #define PCIE_RC_PL_STATS_RXTLP_HI 0x1960 #define PCIE_RC_PL_STATS_RXDLLP_LO 0x1964 #define PCIE_RC_PL_STATS_RXDLLP_HI 0x1968 #define PCIE_RC_PL_STATS_RXPL_ERR 0x1974 #define PCIE_RC_PL_STATS_RXDL_ERR 0x1978 #define PCIE_RC_PL_STATS_RXTL_ERR 0x197c #define PCIE_RC_PL_LTSSM_STATS_3 0x19b0 #define PCIE_RC_PL_LTSSM_STATS_3_TIME_L0S_MASK 0xffff0000 #define PCIE_RC_PL_LTSSM_STATS_3_TIME_RECOV_MASK 0x0000ffff #define PCIE_RC_PL_LTSSM_STATS_CNT 0x19b4 #define PCIE_RC_PL_LTSSM_STATS_CNT_L0S_FAIL_MASK 0xffff0000 #define PCIE_RC_PL_LTSSM_STATS_CNT_RECOV_MASK 0x0000ffff #define PCIE_RC_PL_LTSSM_HIST_0 0x1cec #define PCIE_RC_PL_LTSSM_HIST(n) (PCIE_RC_PL_LTSSM_HIST_0 + ((n) * 4)) #define PCIE_MISC_MISC_CTRL 0x4008 #define PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK 0x80 #define PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK 0x400 #define PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK 0x1000 #define PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK 0x2000 #define PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK 0x300000 #define PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK 0xf8000000 #define PCIE_MISC_MISC_CTRL_SCB1_SIZE_MASK 0x07c00000 #define PCIE_MISC_MISC_CTRL_SCB2_SIZE_MASK 0x0000001f #define SCB_SIZE_MASK(x) PCIE_MISC_MISC_CTRL_SCB ## x ## _SIZE_MASK #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO 0x400c #define PCIE_MEM_WIN0_LO(win) PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO + ((win) * 8) #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI 0x4010 #define PCIE_MEM_WIN0_HI(win) PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 8) #define PCIE_BRCM_MAX_INBOUND_WINS 16 #define PCIE_MISC_RC_BAR1_CONFIG_LO 0x402c #define PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK 0x1f #define PCIE_MISC_RC_BAR4_CONFIG_LO 0x40d4 #define PCIE_MISC_MSI_BAR_CONFIG_LO 0x4044 #define PCIE_MISC_MSI_BAR_CONFIG_HI 0x4048 #define PCIE_MISC_MSI_DATA_CONFIG 0x404c #define PCIE_MISC_MSI_DATA_CONFIG_VAL_32 0xffe06540 #define PCIE_MISC_MSI_DATA_CONFIG_VAL_8 0xfff86540 #define PCIE_MISC_RC_CONFIG_RETRY_TIMEOUT 0x405c #define PCIE_MISC_PCIE_CTRL 0x4064 #define PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK 0x1 #define PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK 0x4 #define PCIE_MISC_PCIE_STATUS 0x4068 #define PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK 0x80 #define PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK 0x20 #define PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK 0x10 #define PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK 0x40 #define PCIE_MISC_REVISION 0x406c #define BRCM_PCIE_HW_REV_33 0x0303 #define BRCM_PCIE_HW_REV_3_20 0x0320 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT 0x4070 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK 0xfff00000 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK 0xfff0 #define PCIE_MEM_WIN0_BASE_LIMIT(win) PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT + ((win) * 4) #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI 0x4080 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK 0xff #define PCIE_MEM_WIN0_BASE_HI(win) PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI + ((win) * 8) #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI 0x4084 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK 0xff #define PCIE_MEM_WIN0_LIMIT_HI(win) PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI + ((win) * 8) #define PCIE_MISC_HARD_PCIE_HARD_DEBUG pcie->soc_data->reg_offsets[PCIE_HARD_DEBUG] #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK 0x2 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_PERST_ASSERT_MASK 0x8 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_REFCLK_OVRD_ENABLE_MASK 0x10000 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_REFCLK_OVRD_OUT_MASK 0x100000 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK 0x200000 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x08000000 #define PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x00800000 #define PCIE_CLKREQ_MASK \ (PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK | \ PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK) /* PCIe Link Capabilities: L0s Entry Supported (PCI Express Base Spec) */ #ifndef PCI_EXP_LNKCAP_ASPM_L0S #define PCI_EXP_LNKCAP_ASPM_L0S 0x00000400 #endif #define PCIE_MISC_CTRL_1 0x40A0 #define PCIE_MISC_CTRL_1_OUTBOUND_TC_MASK 0xf #define PCIE_MISC_CTRL_1_OUTBOUND_NO_SNOOP_MASK RT_BIT(3) #define PCIE_MISC_CTRL_1_OUTBOUND_RO_MASK RT_BIT(4) #define PCIE_MISC_CTRL_1_EN_VDM_QOS_CONTROL_MASK RT_BIT(5) #define PCIE_MISC_UBUS_CTRL 0x40a4 #define PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_ERR_DIS_MASK RT_BIT(13) #define PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_DECERR_DIS_MASK RT_BIT(19) #define PCIE_MISC_UBUS_TIMEOUT 0x40a8 #define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP 0x40ac #define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_ENABLE_MASK RT_BIT(0) #define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_HI 0x40b0 #define PCIE_MISC_UBUS_BAR4_CONFIG_REMAP 0x410c #define PCIE_MISC_UBUS_BAR2_CONFIG_REMAP 0x40b4 #define PCIE_MISC_UBUS_BAR2_CONFIG_REMAP_ACCESS_ENABLE_MASK RT_BIT(0) #define PCIE_MISC_UBUS_BAR3_CONFIG_REMAP 0x40bc /* Additional RC BARs */ #define PCIE_MISC_RC_BAR_CONFIG_LO_SIZE_MASK 0x1f #define PCIE_MISC_RC_BAR4_CONFIG_LO 0x40d4 #define PCIE_MISC_RC_BAR4_CONFIG_HI 0x40d8 #define PCIE_MISC_RC_BAR10_CONFIG_LO 0x4104 #define PCIE_MISC_RC_BAR10_CONFIG_HI 0x4108 #define PCIE_MISC_UBUS_BAR_CONFIG_REMAP_ENABLE 0x1 #define PCIE_MISC_UBUS_BAR_CONFIG_REMAP_LO_MASK 0xfffff000 #define PCIE_MISC_UBUS_BAR_CONFIG_REMAP_HI_MASK 0xff #define PCIE_MISC_UBUS_BAR4_CONFIG_REMAP_LO 0x410c #define PCIE_MISC_UBUS_BAR4_CONFIG_REMAP_HI 0x4110 #define PCIE_MISC_UBUS_BAR10_CONFIG_REMAP_LO 0x413c #define PCIE_MISC_UBUS_BAR10_CONFIG_REMAP_HI 0x4140 /* AXI priority forwarding - automatic level-based */ #define PCIE_MISC_TC_QUEUE_TO_QOS_MAP(x) (0x4160 - (x) * 4) /* Defined in quarter-fullness */ #define QUEUE_THRESHOLD_34_TO_QOS_MAP_SHIFT 12 #define QUEUE_THRESHOLD_23_TO_QOS_MAP_SHIFT 8 #define QUEUE_THRESHOLD_12_TO_QOS_MAP_SHIFT 4 #define QUEUE_THRESHOLD_01_TO_QOS_MAP_SHIFT 0 #define QUEUE_THRESHOLD_MASK 0xf /* VDM messages indexing TCs to AXI priorities */ /* Indexes 8-15 */ #define PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_HI 0x4164 /* Indexes 0-7 */ #define PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_LO 0x4168 #define VDM_PRIORITY_TO_QOS_MAP_SHIFT(x) (4 * (x)) #define VDM_PRIORITY_TO_QOS_MAP_MASK 0xf #define PCIE_MISC_AXI_INTF_CTRL 0x416c #define AXI_EN_RCLK_QOS_ARRAY_FIX RT_BIT(13) #define AXI_EN_QOS_UPDATE_TIMING_FIX RT_BIT(12) #define AXI_DIS_QOS_GATING_IN_MASTER RT_BIT(11) #define AXI_REQFIFO_EN_QOS_PROPAGATION RT_BIT(7) #define AXI_BRIDGE_LOW_LATENCY_MODE RT_BIT(6) #define AXI_MASTER_MAX_OUTSTANDING_REQUESTS_MASK 0x3f #define PCIE_MISC_AXI_READ_ERROR_DATA 0x4170 #define PCIE_MSI_INTR2_BASE 0x4500 /* Offsets from PCIE_INTR2_CPU_BASE and PCIE_MSI_INTR2_BASE */ #define MSI_INT_STATUS 0x0 #define MSI_INT_CLR 0x8 #define MSI_INT_MASK_SET 0x10 #define MSI_INT_MASK_CLR 0x14 #define PCIE_EXT_CFG_DATA 0x8000 #define PCIE_EXT_CFG_INDEX 0x9000 #define PCIE_RGR1_SW_INIT_1_PERST_MASK 0x1 #define PCIE_RGR1_SW_INIT_1_PERST_SHIFT 0x0 #define RGR1_SW_INIT_1_INIT_GENERIC_MASK 0x2 #define RGR1_SW_INIT_1_INIT_GENERIC_SHIFT 0x1 #define RGR1_SW_INIT_1_INIT_7278_MASK 0x1 #define RGR1_SW_INIT_1_INIT_7278_SHIFT 0x0 /* PCIe parameters */ #define BRCM_NUM_PCIE_OUT_WINS 0x4 #define BRCM_INT_PCI_MSI_NR 32 #define BRCM_INT_PCI_MSI_LEGACY_NR 8 #define BRCM_INT_PCI_MSI_SHIFT 0 #define BRCM_INT_PCI_MSI_MASK RT_GENMASK(BRCM_INT_PCI_MSI_NR - 1, 0) #define BRCM_INT_PCI_MSI_LEGACY_MASK RT_GENMASK(31, 32 - BRCM_INT_PCI_MSI_LEGACY_NR) /* MSI target addresses */ #define BRCM_MSI_TARGET_ADDR_LT_4GB 0x0fffffffcULL #define BRCM_MSI_TARGET_ADDR_GT_4GB 0xffffffffcULL /* PCIe CEM 2.2 / PCIe r5.0 6.6.1: wait after PERST# deassertion */ #define PCIE_RESET_CONFIG_WAIT_MS 100 /* MDIO registers */ #define MDIO_PORT0 0x0 #define MDIO_DATA_MASK 0x7fffffff #define MDIO_PORT_MASK 0xf0000 #define MDIO_PORT_EXT_MASK 0x200000 #define MDIO_REGAD_MASK 0xffff #define MDIO_CMD_MASK 0x00100000 #define MDIO_CMD_READ 0x1 #define MDIO_CMD_WRITE 0x0 #define MDIO_DATA_DONE_MASK 0x80000000 #define MDIO_RD_DONE(x) (((x) & MDIO_DATA_DONE_MASK) ? 1 : 0) #define MDIO_WT_DONE(x) (((x) & MDIO_DATA_DONE_MASK) ? 0 : 1) #define SSC_REGS_ADDR 0x1100 #define SET_ADDR_OFFSET 0x1f #define SSC_CNTL_OFFSET 0x2 #define SSC_CNTL_OVRD_EN_MASK 0x8000 #define SSC_CNTL_OVRD_VAL_MASK 0x4000 #define SSC_STATUS_OFFSET 0x1 #define SSC_STATUS_SSC_MASK 0x400 #define SSC_STATUS_PLL_LOCK_MASK 0x800 #define PCIE_BRCM_MAX_MEMC 3 #define IDX_ADDR(pcie) (pcie->soc_data->reg_offsets[EXT_CFG_INDEX]) #define DATA_ADDR(pcie) (pcie->soc_data->reg_offsets[EXT_CFG_DATA]) #define HARD_DEBUG(pcie) (pcie->soc_data->reg_offsets[PCIE_HARD_DEBUG]) #define INTR2_CPU_BASE(pcie) (pcie->soc_data->reg_offsets[PCIE_INTR2_CPU_BASE]) #define PCIE_RGR1_SW_INIT_1(pcie) (pcie->soc_data->reg_offsets[RGR1_SW_INIT_1]) /* Rescal registers */ #define PCIE_DVT_PMU_PHY_CTRL 0xc700 #define PCIE_DVT_PMU_PHY_CTRL_DAST_NFLDS 0x3 #define PCIE_DVT_PMU_PHY_CTRL_DAST_DIG_RESET_MASK 0x4 #define PCIE_DVT_PMU_PHY_CTRL_DAST_DIG_RESET_SHIFT 0x2 #define PCIE_DVT_PMU_PHY_CTRL_DAST_RESET_MASK 0x2 #define PCIE_DVT_PMU_PHY_CTRL_DAST_RESET_SHIFT 0x1 #define PCIE_DVT_PMU_PHY_CTRL_DAST_PWRDN_MASK 0x1 #define PCIE_DVT_PMU_PHY_CTRL_DAST_PWRDN_SHIFT 0x0 enum { RGR1_SW_INIT_1, EXT_CFG_INDEX, EXT_CFG_DATA, PCIE_HARD_DEBUG, PCIE_INTR2_CPU_BASE, }; enum { RGR1_SW_INIT_1_INIT_MASK, RGR1_SW_INIT_1_INIT_SHIFT, }; enum brcm_pcie_type { GENERIC, BCM2711, BCM4908, BCM7278, BCM7425, BCM7435, BCM7712, }; static const char * const supplies_prefix[] = { "vpcie3v3", "vpcie3v3aux", "vpcie12v", }; struct brcm_pcie; struct inbound_win { rt_uint64_t size; rt_uint64_t pci_offset; rt_uint64_t cpu_addr; }; struct pcie_cfg_soc_data { const int *reg_offsets; const enum brcm_pcie_type type; const rt_bool_t has_phy; #define CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN RT_BIT(0) #define CFG_QUIRK_NO_SSC RT_BIT(1) const rt_uint32_t quirks; rt_uint8_t num_inbound_wins; rt_err_t (*perst_set)(struct brcm_pcie *pcie, rt_uint32_t val); rt_err_t (*bridge_sw_init_set)(struct brcm_pcie *pcie, rt_uint32_t val); rt_err_t (*post_setup)(struct brcm_pcie *pcie); }; struct brcm_msi { int irq; void *pcie_base; void *intr_base; struct rt_pic inner_pic; rt_uint64_t data_address; rt_bool_t legacy; /* Some chips have MSIs in bits [31..24] of a shared register. */ int legacy_shift; int msi_max_nr; RT_BITMAP_DECLARE(msi_map, 64); struct rt_spinlock lock; }; /* Internal PCIe Host Controller Information. */ struct brcm_pcie { const struct pcie_cfg_soc_data *soc_data; struct rt_device *dev; void *base; rt_bool_t ssc; /* Spread Spectrum Clocking */ rt_bool_t l1ss; /* L1 PM Substates */ rt_bool_t ep_wakeup_capable; rt_uint32_t qos_map; rt_uint32_t generation; rt_uint32_t misc_revision; rt_uint32_t tperst_clk_ms; rt_size_t memc_nr; rt_uint64_t memc_size[PCIE_BRCM_MAX_MEMC]; struct rt_clk *clk; struct rt_reset_control *rescal_rstc; struct rt_reset_control *perst_rstc; struct rt_reset_control *bridge_rstc; struct rt_reset_control *swinit_rstc; struct rt_regulator *supplies[RT_ARRAY_SIZE(supplies_prefix)]; struct brcm_msi *msi; rt_uint64_t msi_data_address; struct rt_pci_host_bridge *bridge; struct inbound_win inbound_wins[PCIE_BRCM_MAX_INBOUND_WINS]; rt_uint8_t num_inbound_wins; }; static void brcm_pcie_reapply_7712_maps(struct brcm_pcie *pcie); rt_inline void replace_u32_bitfield(rt_uint32_t *p, rt_uint32_t val, rt_uint32_t field) { *p = (*p & ~field) | ((val & (field / (field & -field))) * (field & -field)); } rt_inline void replace_u16_bitfield(rt_uint16_t *p, rt_uint16_t val, rt_uint16_t field) { *p = (*p & ~field) | ((val & (field / (field & -field))) * (field & -field)); } rt_inline int __fls64(rt_uint64_t x) { const rt_uint64_t bits_per_u64 = (sizeof(x) * 8); int num = bits_per_u64 - 1; if (x == 0) { return 0; } if (!(x & (~0ULL << (bits_per_u64 - 32)))) { num -= 32; x <<= 32; } if (!(x & (~0ULL << (bits_per_u64 - 16)))) { num -= 16; x <<= 16; } if (!(x & (~0ULL << (bits_per_u64 - 8)))) { num -= 8; x <<= 8; } if (!(x & (~0ULL << (bits_per_u64 - 4)))) { num -= 4; x <<= 4; } if (!(x & (~0ULL << (bits_per_u64 - 2)))) { num -= 2; x <<= 2; } if (!(x & (~0ULL << (bits_per_u64 - 1)))) { num -= 1; } return num + 1; } rt_uint32_t __hweight32(rt_uint32_t w) { rt_uint32_t res = w - ((w >> 1) & 0x55555555); res = (res & 0x33333333) + ((res >> 2) & 0x33333333); res = (res + (res >> 4)) & 0x0f0f0f0f; res = res + (res >> 8); return (res + (res >> 16)) & 0x000000ff; } rt_inline rt_bool_t is_bmips(const struct brcm_pcie *pcie) { const struct pcie_cfg_soc_data *soc_data = pcie->soc_data; return soc_data->type == BCM7435 || soc_data->type == BCM7425; } /* * This is to convert the size of the inbound "BAR" region to the * non-linear values of PCIE_X_MISC_RC_BAR[123]_CONFIG_LO.SIZE */ static int brcm_pcie_encode_ibar_size(rt_uint64_t size) { int log2_in = rt_ilog2(size); if (log2_in >= 12 && log2_in <= 15) { /* Covers 4KB to 32KB (inclusive) */ return (log2_in - 12) + 0x1c; } else if (log2_in >= 16 && log2_in <= 36) { /* Covers 64KB to 64GB, (inclusive) */ return log2_in - 15; } /* Something is awry so disable */ return 0; } static rt_uint32_t brcm_pcie_mdio_form_pkt(int port, int regad, int cmd) { rt_uint32_t pkt = 0; pkt |= RT_FIELD_PREP(MDIO_PORT_EXT_MASK, port >> 4); pkt |= RT_FIELD_PREP(MDIO_PORT_MASK, port); pkt |= RT_FIELD_PREP(MDIO_REGAD_MASK, regad); pkt |= RT_FIELD_PREP(MDIO_CMD_MASK, cmd); return pkt; } /* Negative return value indicates error */ static rt_err_t brcm_pcie_mdio_read(void *base, rt_uint8_t port, rt_uint8_t regad, rt_uint32_t *val) { rt_uint32_t data; HWREG32(base + PCIE_RC_DL_MDIO_ADDR) = brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_READ); data = HWREG32(base + PCIE_RC_DL_MDIO_ADDR); data = HWREG32(base + PCIE_RC_DL_MDIO_RD_DATA); for (int tries = 0; !MDIO_RD_DONE(data) && tries < 10; ++tries) { rt_hw_us_delay(10); data = HWREG32(base + PCIE_RC_DL_MDIO_RD_DATA); } *val = RT_FIELD_GET(MDIO_DATA_MASK, data); return MDIO_RD_DONE(data) ? 0 : -RT_EIO; } /* Negative return value indicates error */ static rt_err_t brcm_pcie_mdio_write(void *base, rt_uint8_t port, rt_uint8_t regad, rt_uint16_t wrdata) { rt_uint32_t data; HWREG32(base + PCIE_RC_DL_MDIO_ADDR) = brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_WRITE); data = HWREG32(base + PCIE_RC_DL_MDIO_ADDR); HWREG32(base + PCIE_RC_DL_MDIO_WR_DATA) = MDIO_DATA_DONE_MASK | wrdata; data = HWREG32(base + PCIE_RC_DL_MDIO_WR_DATA); for (int tries = 0; !MDIO_WT_DONE(data) && tries < 10; ++tries) { rt_hw_us_delay(10); data = HWREG32(base + PCIE_RC_DL_MDIO_WR_DATA); } return MDIO_WT_DONE(data) ? 0 : -RT_EIO; } /* * Configures device for Spread Spectrum Clocking (SSC) mode; * a negative return value indicates error. */ static rt_err_t brcm_pcie_set_ssc(struct brcm_pcie *pcie) { int pll, ssc; rt_err_t err; rt_uint32_t tmp; if ((err = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET, SSC_REGS_ADDR)) < 0) { return err; } if ((err = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0, SSC_CNTL_OFFSET, &tmp)) < 0) { return err; } replace_u32_bitfield(&tmp, 1, SSC_CNTL_OVRD_EN_MASK); replace_u32_bitfield(&tmp, 1, SSC_CNTL_OVRD_VAL_MASK); if ((err = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SSC_CNTL_OFFSET, tmp)) < 0) { return err; } rt_hw_us_delay(1500); if ((err = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0, SSC_STATUS_OFFSET, &tmp)) < 0) { return err; } ssc = RT_FIELD_GET(SSC_STATUS_SSC_MASK, tmp); pll = RT_FIELD_GET(SSC_STATUS_PLL_LOCK_MASK, tmp); return ssc && pll ? 0 : -RT_EIO; } /* * SerDes MDIO refclk programming for BCM7712/2712 (54MHz xosc). * Must run after SERDES_IDDQ is cleared. */ static void brcm_pcie_delay_ms(rt_uint32_t ms); static rt_err_t brcm_pcie_serdes_munge_pll(struct brcm_pcie *pcie) { void *base = pcie->base; const rt_uint16_t data[] = { 0x50b9, 0xbda1, 0x0094, 0x97b4, 0x5030, 0x5030, 0x0007 }; const rt_uint8_t regs[] = { 0x16, 0x17, 0x18, 0x19, 0x1b, 0x1c, 0x1e }; rt_err_t err; if ((err = brcm_pcie_mdio_write(base, MDIO_PORT0, SET_ADDR_OFFSET, 0x1600))) { return err; } for (int i = 0; i < RT_ARRAY_SIZE(regs); ++i) { if ((err = brcm_pcie_mdio_write(base, MDIO_PORT0, regs[i], data[i]))) { return err; } } rt_hw_us_delay(200); return RT_EOK; } static void brcm_pcie_serdes_pm_clk_period(struct brcm_pcie *pcie) { rt_uint32_t tmp; void *base = pcie->base; /* * L1SS sub-state timers: PM clock period 18.52ns (1/54MHz, round down). */ tmp = HWREG32(base + PCIE_RC_PL_PHY_CTL_15); tmp &= ~PCIE_RC_PL_PHY_CTL_15_PM_CLK_PERIOD_MASK; tmp |= 0x12; HWREG32(base + PCIE_RC_PL_PHY_CTL_15) = tmp; } /* Limits operation to a specific generation (1, 2, or 3) */ static void brcm_pcie_set_generation(struct brcm_pcie *pcie, int generation) { rt_uint16_t lnkctl2; rt_uint32_t lnkcap; lnkctl2 = HWREG16(pcie->base + BRCM_PCIE_CAP_REGS + PCIER_LINK_CTL2); lnkcap = HWREG32(pcie->base + BRCM_PCIE_CAP_REGS + PCIER_LINK_CAP); replace_u32_bitfield(&lnkcap, generation, PCIEM_LINK_CAP_MAX_SPEED); HWREG32(pcie->base + BRCM_PCIE_CAP_REGS + PCIER_LINK_CAP) = lnkcap; replace_u16_bitfield(&lnkctl2, generation, PCIEM_LNKCTL2_TLS); HWREG16(pcie->base + BRCM_PCIE_CAP_REGS + PCIER_LINK_CTL2) = lnkctl2; } static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie, rt_uint32_t win, rt_uint64_t cpu_addr, rt_uint64_t pcie_addr, rt_uint64_t size) { int high_addr_shift; rt_ubase_t cpu_addr_mb, limit_addr_mb; rt_uint32_t cpu_addr_mb_high, limit_addr_mb_high, tmp; /* Set the base of the pcie_addr window */ HWREG32(pcie->base + PCIE_MEM_WIN0_LO(win)) = rt_lower_32_bits(pcie_addr); HWREG32(pcie->base + PCIE_MEM_WIN0_HI(win)) = rt_upper_32_bits(pcie_addr); /* Write the addr base & limit lower bits (in MBs) */ cpu_addr_mb = cpu_addr / SIZE_MB; limit_addr_mb = (cpu_addr + size - 1) / SIZE_MB; tmp = HWREG32(pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win)); replace_u32_bitfield(&tmp, cpu_addr_mb, PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK); replace_u32_bitfield(&tmp, limit_addr_mb, PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK); HWREG32(pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win)) = tmp; if (is_bmips(pcie)) { return; } /* Write the cpu & limit addr upper bits */ high_addr_shift = __hweight32(PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK); cpu_addr_mb_high = cpu_addr_mb >> high_addr_shift; tmp = HWREG32(pcie->base + PCIE_MEM_WIN0_BASE_HI(win)); replace_u32_bitfield(&tmp, cpu_addr_mb_high, PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK); HWREG32(pcie->base + PCIE_MEM_WIN0_BASE_HI(win)) = tmp; limit_addr_mb_high = limit_addr_mb >> high_addr_shift; tmp = HWREG32(pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win)); replace_u32_bitfield(&tmp, limit_addr_mb_high, PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK); HWREG32(pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win)) = tmp; } static void brcm_pcie_misc_ctrl_enable_access(struct brcm_pcie *pcie) { void *base = pcie->base; rt_uint32_t tmp = HWREG32(base + PCIE_MISC_MISC_CTRL); rt_uint32_t burst; if (is_bmips(pcie)) { burst = 0x1; } else if (pcie->soc_data->type == BCM2711) { burst = 0x0; } else if (pcie->soc_data->type == BCM7278) { burst = 0x3; } else { burst = 0x2; } replace_u32_bitfield(&tmp, 1, PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK); replace_u32_bitfield(&tmp, 1, PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK); replace_u32_bitfield(&tmp, burst, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK); replace_u32_bitfield(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK); replace_u32_bitfield(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK); HWREG32(base + PCIE_MISC_MISC_CTRL) = tmp; } static void brcm_pcie_apply_outbound_wins(struct brcm_pcie *pcie) { struct rt_pci_host_bridge *bridge = pcie->bridge; int num_out_wins = 0; for (int i = 0; i < bridge->bus_regions_nr; ++i) { struct rt_pci_bus_region *region = &bridge->bus_regions[i]; if (region->flags != PCI_BUS_REGION_F_MEM && region->flags != PCI_BUS_REGION_F_PREFETCH) { continue; } if (num_out_wins >= BRCM_NUM_PCIE_OUT_WINS) { LOG_E("Too many outbound wins"); return; } if (is_bmips(pcie)) { rt_uint64_t start = region->cpu_addr; rt_off_t offset = region->cpu_addr - region->phy_addr; rt_uint32_t nwins = region->size / 128 * SIZE_MB; if (nwins > BRCM_NUM_PCIE_OUT_WINS) { nwins = BRCM_NUM_PCIE_OUT_WINS; } for (int j = 0; j < nwins; ++j, start += 128 * SIZE_MB) { brcm_pcie_set_outbound_win(pcie, j, start, start - offset, 128 * SIZE_MB); } return; } brcm_pcie_set_outbound_win(pcie, num_out_wins, region->cpu_addr, region->phy_addr, region->size); ++num_out_wins; } } /* The controller is capable of serving in both RC and EP roles */ static rt_bool_t brcm_pcie_rc_mode(struct brcm_pcie *pcie) { void *base = pcie->base; rt_uint32_t val = HWREG32(base + PCIE_MISC_PCIE_STATUS); return !!RT_FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK, val); } static rt_bool_t brcm_pcie_link_up(struct brcm_pcie *pcie) { rt_uint32_t val = HWREG32(pcie->base + PCIE_MISC_PCIE_STATUS); rt_uint32_t dla = RT_FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK, val); rt_uint32_t plu = RT_FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK, val); return dla && plu; } static void brcm_pcie_prep_train_clk(struct brcm_pcie *pcie) { rt_uint32_t tmp; void *base = pcie->base; tmp = HWREG32(base + HARD_DEBUG(pcie)); tmp &= ~PCIE_CLKREQ_MASK; HWREG32(base + HARD_DEBUG(pcie)) = tmp; } static void brcm_pcie_delay_ms(rt_uint32_t ms) { if (ms == 0) { return; } /* * Always busy-wait: link training runs during init (often before the * scheduler is fully ready). mdelay can yield or mis-time here. */ rt_hw_us_delay(ms * 1000UL); } static void *brcm_pcie_map(struct rt_pci_bus *bus, rt_uint32_t devfn, int reg) { void *base; struct brcm_pcie *pcie = bus->sysdata; base = pcie->base; /* Accesses to the RC go right to the RC registers if !devfn */ if (rt_pci_is_root_bus(bus)) { return devfn ? RT_NULL : base + PCIE_ECAM_REG(reg); } /* An access to our HW w/o link-up will cause a CPU Abort */ if (!brcm_pcie_link_up(pcie)) { return RT_NULL; } /* For devices, write to the config space index register */ HWREG32(pcie->base + IDX_ADDR(pcie)) = PCIE_ECAM_OFFSET(bus->number, devfn, 0); return base + DATA_ADDR(pcie) + PCIE_ECAM_REG(reg); } static void *brcm7425_pcie_map(struct rt_pci_bus *bus, rt_uint32_t devfn, int reg) { void *base; struct brcm_pcie *pcie = bus->sysdata; base = pcie->base; /* Accesses to the RC go right to the RC registers if !devfn */ if (rt_pci_is_root_bus(bus)) { return devfn ? RT_NULL : base + PCIE_ECAM_REG(reg); } /* An access to our HW w/o link-up will cause a CPU Abort */ if (!brcm_pcie_link_up(pcie)) { return RT_NULL; } /* For devices, write to the config space index register */ HWREG32(base + IDX_ADDR(pcie)) = PCIE_ECAM_OFFSET(bus->number, devfn, reg); return base + DATA_ADDR(pcie); } static rt_err_t brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, rt_uint32_t val) { rt_err_t err; rt_uint32_t tmp, mask = RGR1_SW_INIT_1_INIT_GENERIC_MASK; rt_uint32_t shift = RGR1_SW_INIT_1_INIT_GENERIC_SHIFT; if (pcie->bridge_rstc) { if (val) { err = rt_reset_control_assert(pcie->bridge_rstc); } else { err = rt_reset_control_deassert(pcie->bridge_rstc); } return err; } tmp = HWREG32(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); tmp = (tmp & ~mask) | ((val << shift) & mask); HWREG32(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)) = tmp; return RT_EOK; } static rt_err_t brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, rt_uint32_t val) { rt_uint32_t tmp, mask = RGR1_SW_INIT_1_INIT_7278_MASK; rt_uint32_t shift = RGR1_SW_INIT_1_INIT_7278_SHIFT; tmp = HWREG32(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); tmp = (tmp & ~mask) | ((val << shift) & mask); HWREG32(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)) = tmp; return RT_EOK; } static rt_err_t brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, rt_uint32_t val) { rt_err_t err; if (!pcie->perst_rstc) { return -RT_EINVAL; } if (val) { err = rt_reset_control_assert(pcie->perst_rstc); } else { err = rt_reset_control_deassert(pcie->perst_rstc); } return err; } static rt_err_t brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, rt_uint32_t val) { rt_uint32_t tmp; /* Perst bit has moved and assert value is 0 */ tmp = HWREG32(pcie->base + PCIE_MISC_PCIE_CTRL); replace_u32_bitfield(&tmp, !val, PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK); HWREG32(pcie->base + PCIE_MISC_PCIE_CTRL) = tmp; return RT_EOK; } static rt_err_t brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, rt_uint32_t val) { rt_uint32_t tmp; tmp = HWREG32(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); replace_u32_bitfield(&tmp, val, PCIE_RGR1_SW_INIT_1_PERST_MASK); HWREG32(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)) = tmp; return RT_EOK; } /* * access timeout may occur during L1SS sleep periods, even without the * presence of a PCIe access. */ static void brcm_extend_rbus_timeout(struct brcm_pcie *pcie) { /* 4 seconds, our setting for L1SS */ const rt_uint32_t timeout_us = 4000000; /* 7712 does not have this (RGR1) timer */ if (pcie->soc_data->type == BCM7712) { return; } /* Each unit in timeout register is 1/216,000,000 seconds */ HWREG32(pcie->base + PCIE_RGR1_SW_INIT_1(pcie) - 8) = 216 * timeout_us; } static void brcm_config_clkreq(struct brcm_pcie *pcie) { rt_err_t err; void *base = pcie->base; rt_uint32_t clkreq_cntl, tmp; const char *mode = "default"; err = rt_dm_dev_prop_read_string(pcie->dev, "brcm,clkreq-mode", &mode); if (err && err != -RT_EINVAL) { mode = "safe"; } /* * brcm,enable-l1ss in DT documents capability; until ASPM is fully * handled in RT-Thread, force refclk on to avoid EP stuck in L1.x. */ if (pcie->soc_data->type == BCM7712 && rt_dm_dev_prop_read_bool(pcie->dev, "brcm,enable-l1ss")) { mode = "safe"; } /* Start out assuming safe mode (both mode bits cleared) */ clkreq_cntl = HWREG32(base + HARD_DEBUG(pcie)); clkreq_cntl &= ~PCIE_CLKREQ_MASK; if (!rt_strcmp(mode, "no-l1ss")) { /* * "no-l1ss" -- Provides Clock Power Management, L0s, and * L1, but cannot provide L1 substate (L1SS) power * savings. If the downstream device connected to the RC is * L1SS capable AND the OS enables L1SS, all PCIe traffic * may abruptly halt, potentially hanging the system. */ clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK; /* * We want to un-advertise L1 substates because if the OS * tries to configure the controller into using L1 substate * power savings it may fail or hang when the RC HW is in * "no-l1ss" mode. */ tmp = HWREG32(base + PCIE_RC_CFG_PRIV1_ROOT_CAP); replace_u32_bitfield(&tmp, 2, PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK); HWREG32(base + PCIE_RC_CFG_PRIV1_ROOT_CAP) = tmp; } else if (!rt_strcmp(mode, "default")) { /* * "default" -- Provides L0s, L1, and L1SS, but not * compliant to provide Clock Power Management; * specifically, may not be able to meet the Tclron max * timing of 400ns as specified in "Dynamic Clock Control", * section 3.2.5.2.2 of the PCIe spec. This situation is * atypical and should happen only with older devices. */ clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK; brcm_extend_rbus_timeout(pcie); } else { /* * "safe" -- No power savings; refclk is driven by RC * unconditionally. */ if (rt_strcmp(mode, "safe") != 0) { LOG_W("invalid 'brcm,clkreq-mode' DT string"); } mode = "safe"; clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_REFCLK_OVRD_OUT_MASK; clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_REFCLK_OVRD_ENABLE_MASK; tmp = HWREG32(base + PCIE_RC_CFG_PRIV1_ROOT_CAP); replace_u32_bitfield(&tmp, 2, PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK); HWREG32(base + PCIE_RC_CFG_PRIV1_ROOT_CAP) = tmp; } HWREG32(base + HARD_DEBUG(pcie)) = clkreq_cntl; } static rt_err_t brcm_pcie_start_link(struct brcm_pcie *pcie) { rt_err_t err; rt_uint32_t tmp; rt_uint16_t tmp16; void *base = pcie->base; if (pcie->generation) { brcm_pcie_set_generation(pcie, pcie->generation); } brcm_pcie_prep_train_clk(pcie); if (pcie->tperst_clk_ms) { tmp = HWREG32(base + HARD_DEBUG(pcie)); replace_u32_bitfield(&tmp, 1, PCIE_MISC_HARD_PCIE_HARD_DEBUG_PERST_ASSERT_MASK); HWREG32(base + HARD_DEBUG(pcie)) = tmp; if ((err = pcie->soc_data->perst_set(pcie, 0))) { return err; } brcm_pcie_delay_ms(pcie->tperst_clk_ms); tmp = HWREG32(base + HARD_DEBUG(pcie)); replace_u32_bitfield(&tmp, 0, PCIE_MISC_HARD_PCIE_HARD_DEBUG_PERST_ASSERT_MASK); HWREG32(base + HARD_DEBUG(pcie)) = tmp; } else { if ((err = pcie->soc_data->perst_set(pcie, 0))) { return err; } } brcm_pcie_delay_ms(PCIE_RESET_CONFIG_WAIT_MS); for (int i = 0; i < 100 && !brcm_pcie_link_up(pcie); i += 5) { brcm_pcie_delay_ms(5); } if (!brcm_pcie_link_up(pcie)) { LOG_E("%s: link down", rt_dm_dev_get_name(pcie->dev)); return -RT_EINVAL; } brcm_config_clkreq(pcie); if (pcie->soc_data->post_setup) { /* * post_setup may run before the link is up; re-apply SerDes PLL * settings after training so downstream refclk is stable. */ if ((err = brcm_pcie_serdes_munge_pll(pcie))) { LOG_W("post-train serdes munge failed: %s", rt_strerror(err)); } else { brcm_pcie_serdes_pm_clk_period(pcie); } } if (pcie->ssc) { if (brcm_pcie_set_ssc(pcie)) { LOG_E("Failed attempt to enter ssc mode"); } } brcm_pcie_reapply_7712_maps(pcie); tmp16 = HWREG16(base + BRCM_PCIE_CAP_REGS + PCIER_ROOT_CAP); if (tmp16 & PCIEM_ROOT_CAP_CRS_VIS) { tmp16 = HWREG16(base + BRCM_PCIE_CAP_REGS + PCIER_ROOT_CTL); replace_u16_bitfield(&tmp16, 1, PCIEM_ROOT_CTL_CRS_VIS); HWREG16(base + BRCM_PCIE_CAP_REGS + PCIER_ROOT_CTL) = tmp16; } return RT_EOK; } static rt_err_t brcm_pcie_add_bus(struct rt_pci_bus *bus) { rt_err_t err = RT_EOK; struct brcm_pcie *pcie = bus->sysdata; struct rt_device *dev = &bus->host_bridge->parent; if (!bus->parent) { return RT_EOK; } if (!rt_pci_is_root_bus(bus->parent)) { return RT_EOK; } if (dev->ofw_node) { rt_size_t supplies = 0; for (int i = 0; i < RT_ARRAY_SIZE(pcie->supplies); ++i) { pcie->supplies[i] = rt_regulator_get(dev, supplies_prefix[i]); if (rt_is_err(pcie->supplies[i])) { err = rt_ptr_err(pcie->supplies[i]); while (i --> 0) { rt_regulator_put(pcie->supplies[i]); pcie->supplies[i] = RT_NULL; } return err; } else if (pcie->supplies[i]) { ++supplies; } } if (!supplies) { goto _no_regulators; } for (int i = 0; i < RT_ARRAY_SIZE(pcie->supplies); ++i) { if (pcie->supplies[i]) { if ((err = rt_regulator_enable(pcie->supplies[i]))) { LOG_E("Can't enable regulators for downstream device"); for (int i = 0; i < RT_ARRAY_SIZE(pcie->supplies); ++i) { rt_regulator_put(pcie->supplies[i]); pcie->supplies[i] = RT_NULL; } return err; } } } } _no_regulators: err = brcm_pcie_start_link(pcie); return err; } static rt_err_t brcm_pcie_remove_bus(struct rt_pci_bus *bus) { struct brcm_pcie *pcie = bus->sysdata; for (int i = 0; i < RT_ARRAY_SIZE(pcie->supplies); ++i) { if (!rt_is_err_or_null(pcie->supplies[i])) { rt_regulator_disable(pcie->supplies[i]); rt_regulator_put(pcie->supplies[i]); } } return RT_EOK; } /* L23 is a low-power PCIe link state */ static void brcm_pcie_enter_l23(struct brcm_pcie *pcie) { int l23; rt_uint32_t tmp; void *base = pcie->base; /* Assert request for L23 */ tmp = HWREG32(base + PCIE_MISC_PCIE_CTRL); replace_u32_bitfield(&tmp, 1, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK); HWREG32(base + PCIE_MISC_PCIE_CTRL) = tmp; /* Wait up to 36 msec for L23 */ tmp = HWREG32(base + PCIE_MISC_PCIE_STATUS); l23 = RT_FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, tmp); for (int i = 0; i < 15 && !l23; ++i) { rt_hw_us_delay(2200); tmp = HWREG32(base + PCIE_MISC_PCIE_STATUS); l23 = RT_FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, tmp); } if (!l23) { LOG_E("%s: failed to enter low-power link state", rt_dm_dev_get_name(pcie->dev)); } } static rt_err_t brcm_phy_cntl(struct brcm_pcie *pcie, const int start) { rt_err_t err = RT_EOK; void *base = pcie->base; rt_uint32_t val, tmp, combined_mask = 0; const int beg = start ? 0 : PCIE_DVT_PMU_PHY_CTRL_DAST_NFLDS - 1; const int end = start ? PCIE_DVT_PMU_PHY_CTRL_DAST_NFLDS : -1; static const rt_uint32_t shifts[PCIE_DVT_PMU_PHY_CTRL_DAST_NFLDS] = { PCIE_DVT_PMU_PHY_CTRL_DAST_PWRDN_SHIFT, PCIE_DVT_PMU_PHY_CTRL_DAST_RESET_SHIFT, PCIE_DVT_PMU_PHY_CTRL_DAST_DIG_RESET_SHIFT, }; static const rt_uint32_t masks[PCIE_DVT_PMU_PHY_CTRL_DAST_NFLDS] = { PCIE_DVT_PMU_PHY_CTRL_DAST_PWRDN_MASK, PCIE_DVT_PMU_PHY_CTRL_DAST_RESET_MASK, PCIE_DVT_PMU_PHY_CTRL_DAST_DIG_RESET_MASK, }; for (int i = beg; i != end; start ? ++i : --i) { val = start ? RT_BIT_MASK(shifts[i]) : 0; tmp = HWREG32(base + PCIE_DVT_PMU_PHY_CTRL); tmp = (tmp & ~masks[i]) | (val & masks[i]); HWREG32(base + PCIE_DVT_PMU_PHY_CTRL) = tmp; rt_hw_us_delay(125); combined_mask |= masks[i]; } tmp = HWREG32(base + PCIE_DVT_PMU_PHY_CTRL); val = start ? combined_mask : 0; if ((tmp & combined_mask) != val) { err = -RT_EIO; LOG_E("%s: failed to %s phy", rt_dm_dev_get_name(pcie->dev), start ? "start" : "stop"); } return err; } static rt_err_t brcm_pcie_post_setup_bcm2712(struct brcm_pcie *pcie) { rt_err_t err; rt_uint32_t tmp; rt_uint8_t qos_map[8]; void *base = pcie->base; if ((err = brcm_pcie_serdes_munge_pll(pcie))) { return err; } rt_hw_us_delay(200); brcm_pcie_serdes_pm_clk_period(pcie); /* * BCM7712/2712 uses a UBUS-AXI bridge. * Suppress AXI error responses and return 1s for read failures. */ tmp = HWREG32(base + PCIE_MISC_UBUS_CTRL); replace_u32_bitfield(&tmp, 1, PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_ERR_DIS_MASK); replace_u32_bitfield(&tmp, 1, PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_DECERR_DIS_MASK); HWREG32(base + PCIE_MISC_UBUS_CTRL) = tmp; HWREG32(base + PCIE_MISC_AXI_READ_ERROR_DATA) = 0xffffffff; /* * Adjust timeouts. The UBUS timeout also affects Configuration Request * Retry responses, as the request will get terminated if * either timeout expires, so both have to be a large value * (in clocks of 750MHz). * Set UBUS timeout to 250ms, then set RC config retry timeout * to be ~240ms. * * If CRSSVE=1 this will stop the core from blocking on a Retry * response, but does require the device to be well-behaved... */ HWREG32(base + PCIE_MISC_UBUS_TIMEOUT) = 0xb2d0000; HWREG32(base + PCIE_MISC_RC_CONFIG_RETRY_TIMEOUT) = 0xaba0000; /* * BCM2712 has a configurable QoS mechanism that assigns TLP Traffic Classes * to separate AXI IDs with a configurable priority scheme. * Dynamic priority elevation is supported through reception of Type 1 * Vendor Defined Messages, but several bugs make this largely ineffective. */ /* Disable broken forwarding search. Set chicken bits for 2712D0 */ tmp = HWREG32(base + PCIE_MISC_AXI_INTF_CTRL); tmp &= ~AXI_REQFIFO_EN_QOS_PROPAGATION; tmp |= AXI_EN_RCLK_QOS_ARRAY_FIX | AXI_EN_QOS_UPDATE_TIMING_FIX | AXI_DIS_QOS_GATING_IN_MASTER; HWREG32(base + PCIE_MISC_AXI_INTF_CTRL) = tmp; /* * Work around spurious QoS=0 assignments to inbound traffic. * If the QOS_UPDATE_TIMING_FIX bit is Reserved-0, then this is a * 2712C1 chip, or a single-lane RC. Use the best-effort alternative * which is to partially throttle AXI requests in-flight to SDRAM. */ tmp = HWREG32(base + PCIE_MISC_AXI_INTF_CTRL); if (!(tmp & AXI_EN_QOS_UPDATE_TIMING_FIX)) { tmp &= ~AXI_MASTER_MAX_OUTSTANDING_REQUESTS_MASK; tmp |= 15; HWREG32(base + PCIE_MISC_AXI_INTF_CTRL) = tmp; } /* Disable VDM reception by default */ tmp = HWREG32(base + PCIE_MISC_CTRL_1); tmp &= ~PCIE_MISC_CTRL_1_EN_VDM_QOS_CONTROL_MASK; HWREG32(base + PCIE_MISC_CTRL_1) = tmp; /* * U-Boot enables BAR2 UBUS remap access for inbound EP MemWr (DMA). * MIP MSIX doorbell (PCI 0xff_ffff_f000) uses inbound BAR3 on 2712. */ tmp = HWREG32(base + PCIE_MISC_UBUS_BAR2_CONFIG_REMAP); replace_u32_bitfield(&tmp, 1, PCIE_MISC_UBUS_BAR2_CONFIG_REMAP_ACCESS_ENABLE_MASK); HWREG32(base + PCIE_MISC_UBUS_BAR2_CONFIG_REMAP) = tmp; tmp = HWREG32(base + PCIE_MISC_UBUS_BAR3_CONFIG_REMAP); replace_u32_bitfield(&tmp, 1, PCIE_MISC_UBUS_BAR2_CONFIG_REMAP_ACCESS_ENABLE_MASK); HWREG32(base + PCIE_MISC_UBUS_BAR3_CONFIG_REMAP) = tmp; if (!rt_dm_dev_prop_read_u8_array_index(pcie->dev, "brcm,fifo-qos-map", 0, 4, qos_map)) { /* * Backpressure mode - each element is QoS for each * quartile of FIFO level. Each TC gets the same map, because * this mode is intended for nonrealtime EPs. */ tmp = 0; for (int i = 0; i < 4; i++) { /* Priorities range from 0-15 */ qos_map[i] &= 0x0f; tmp |= qos_map[i] << (i * 4); } for (int i = 0; i < 8; i++) { HWREG32(base + PCIE_MISC_TC_QUEUE_TO_QOS_MAP(i)) = tmp; } } else if (!rt_dm_dev_prop_read_u8_array_index(pcie->dev, "brcm,vdm-qos-map", 0, 8, qos_map)) { /* Enable VDM reception */ tmp = HWREG32(base + PCIE_MISC_CTRL_1); tmp |= PCIE_MISC_CTRL_1_EN_VDM_QOS_CONTROL_MASK; HWREG32(base + PCIE_MISC_CTRL_1) = tmp; tmp = 0; for (int i = 0; i < 8; i++) { /* Priorities range from 0-15 */ qos_map[i] &= 0x0f; tmp |= qos_map[i] << (i * 4); } /* Broken forwarding means no point separating panic priorities from normal */ HWREG32(base + PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_LO) = tmp; HWREG32(base + PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_HI) = tmp; /* Match Vendor ID of 0 */ HWREG32(base + PCIE_RC_TL_VDM_CTL1) = 0; /* * Forward VDMs to priority interface anyway - * useful for debugging starvation through the received VDM count fields. */ tmp = HWREG32(base + PCIE_RC_TL_VDM_CTL0); tmp |= PCIE_RC_TL_VDM_CTL0_VDM_ENABLED_MASK | PCIE_RC_TL_VDM_CTL0_VDM_IGNORETAG_MASK | PCIE_RC_TL_VDM_CTL0_VDM_IGNOREVNDRID_MASK; HWREG32(base + PCIE_RC_TL_VDM_CTL0) = tmp; } return RT_EOK; } static const int pcie_offsets[] = { [RGR1_SW_INIT_1] = 0x9210, [EXT_CFG_INDEX] = 0x9000, [EXT_CFG_DATA] = 0x8000, [PCIE_HARD_DEBUG] = 0x4204, [PCIE_INTR2_CPU_BASE] = 0x4300, }; static const int pcie_offsets_bcm7278[] = { [RGR1_SW_INIT_1] = 0xc010, [EXT_CFG_INDEX] = 0x9000, [EXT_CFG_DATA] = 0x8000, [PCIE_HARD_DEBUG] = 0x4204, [PCIE_INTR2_CPU_BASE] = 0x4300, }; static const int pcie_offsets_bcm7425[] = { [RGR1_SW_INIT_1] = 0x8010, [EXT_CFG_INDEX] = 0x8300, [EXT_CFG_DATA] = 0x8304, [PCIE_HARD_DEBUG] = 0x4204, [PCIE_INTR2_CPU_BASE] = 0x4300, }; static const int pcie_offsets_bcm7712[] = { [RGR1_SW_INIT_1] = 0x9210, [EXT_CFG_INDEX] = 0x9000, [EXT_CFG_DATA] = 0x8000, [PCIE_HARD_DEBUG] = 0x4304, [PCIE_INTR2_CPU_BASE] = 0x4400, }; static const struct pcie_cfg_soc_data generic_cfg = { .reg_offsets = pcie_offsets, .type = GENERIC, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 3, }; static const struct pcie_cfg_soc_data bcm7425_cfg = { .reg_offsets = pcie_offsets_bcm7425, .type = BCM7425, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 3, }; static const struct pcie_cfg_soc_data bcm7435_cfg = { .reg_offsets = pcie_offsets, .type = BCM7435, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 3, }; static const struct pcie_cfg_soc_data bcm4908_cfg = { .reg_offsets = pcie_offsets, .type = BCM4908, .perst_set = brcm_pcie_perst_set_4908, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 3, }; static const struct pcie_cfg_soc_data bcm7278_cfg = { .reg_offsets = pcie_offsets_bcm7278, .type = BCM7278, .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, .num_inbound_wins = 3, }; static const struct pcie_cfg_soc_data bcm2711_cfg = { .reg_offsets = pcie_offsets, .type = BCM2711, .perst_set = brcm_pcie_perst_set_generic, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 3, }; static const struct pcie_cfg_soc_data bcm7216_cfg = { .reg_offsets = pcie_offsets_bcm7278, .type = BCM7278, .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, .has_phy = RT_TRUE, .num_inbound_wins = 3, }; static const struct pcie_cfg_soc_data bcm7712_cfg = { .reg_offsets = pcie_offsets_bcm7712, .type = BCM7712, .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .num_inbound_wins = 10, }; static const struct pcie_cfg_soc_data bcm2712_cfg = { .reg_offsets = pcie_offsets_bcm7712, .type = BCM7712, .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .post_setup = brcm_pcie_post_setup_bcm2712, .quirks = CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN | CFG_QUIRK_NO_SSC, .num_inbound_wins = 10, }; static const struct rt_pci_ops brcm_pcie_ops = { .add = brcm_pcie_add_bus, .remove = brcm_pcie_remove_bus, .map = brcm_pcie_map, .read = rt_pci_bus_read_config_uxx, .write = rt_pci_bus_write_config_uxx, }; static const struct rt_pci_ops brcm7425_pcie_ops = { .add = brcm_pcie_add_bus, .remove = brcm_pcie_remove_bus, .map = brcm7425_pcie_map, .read = rt_pci_bus_read_config_generic_u32, .write = rt_pci_bus_write_config_generic_u32, }; static void brcm_msi_ack_irq(struct rt_pic_irq *pirq) { struct brcm_msi *msi = pirq->pic->priv_data; HWREG32(msi->intr_base + MSI_INT_CLR) = 1 << (pirq->hwirq + msi->legacy_shift); } static void brcm_msi_compose_msi_msg(struct rt_pic_irq *pirq, struct rt_pci_msi_msg *msg) { struct brcm_msi *msi = pirq->pic->priv_data; msg->address_lo = rt_lower_32_bits(msi->data_address); msg->address_hi = rt_upper_32_bits(msi->data_address); msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | pirq->hwirq; } static int brcm_msi_irq_alloc(struct rt_pic *pic, struct rt_pci_msi_desc *msi_desc) { int irq, hwirq; rt_ubase_t level; struct rt_pic_irq *pirq; struct brcm_msi *msi = pic->priv_data; level = rt_spin_lock_irqsave(&msi->lock); hwirq = rt_bitmap_next_clear_bit(msi->msi_map, 0, msi->msi_max_nr); if (hwirq >= msi->msi_max_nr) { irq = -RT_EEMPTY; goto _out_lock; } rt_bitmap_set_bit(msi->msi_map, hwirq); irq = rt_pic_config_irq(pic, hwirq, hwirq); pirq = rt_pic_find_irq(pic, hwirq); pirq->mode = RT_IRQ_MODE_EDGE_RISING; _out_lock: rt_spin_unlock_irqrestore(&msi->lock, level); return irq; } static void brcm_msi_irq_free(struct rt_pic *pic, int irq) { rt_ubase_t level; struct rt_pic_irq *pirq; struct brcm_msi *msi = pic->priv_data; pirq = rt_pic_find_pirq(pic, irq); if (!pirq) { return; } level = rt_spin_lock_irqsave(&msi->lock); rt_bitmap_clear_bit(msi->msi_map, pirq->hwirq); rt_spin_unlock_irqrestore(&msi->lock, level); } const static struct rt_pic_ops brcm_msi_ops = { .name = "BRCMSTB-MSI", .irq_ack = brcm_msi_ack_irq, .irq_mask = rt_pci_msi_mask_irq, .irq_unmask = rt_pci_msi_unmask_irq, .irq_compose_msi_msg = brcm_msi_compose_msi_msg, .irq_alloc_msi = brcm_msi_irq_alloc, .irq_free_msi = brcm_msi_irq_free, }; static void brcm_pcie_msi_isr(int irqno, void *param) { rt_uint32_t status; rt_uint32_t raw_status; struct rt_pic_irq *pirq; struct brcm_msi *msi = param; RT_UNUSED(irqno); raw_status = HWREG32(msi->intr_base + MSI_INT_STATUS); status = raw_status >> msi->legacy_shift; for (int bit = 0; bit < BRCM_INT_PCI_MSI_NR && status; ++bit) { if (!(RT_BIT(bit) & status)) { continue; } pirq = rt_pic_find_irq(&msi->inner_pic, bit); brcm_msi_ack_irq(pirq); rt_pic_handle_isr(pirq); pirq = rt_pic_find_irq(&msi->inner_pic, bit + 32); brcm_msi_ack_irq(pirq); rt_pic_handle_isr(pirq); status &= ~RT_BIT(bit); } rt_hw_interrupt_umask(msi->irq); } static rt_err_t brcm_pcie_enable_msi(struct brcm_pcie *pcie) { int irq; rt_uint32_t val; struct brcm_msi *msi; struct rt_device *dev = pcie->dev; if ((irq = rt_dm_dev_get_irq(dev, 1)) < 0) { LOG_E("Read map MSI interrupt error = %s", rt_strerror(irq)); return irq; } if (!(msi = rt_calloc(1, sizeof(*msi)))) { return -RT_ENOMEM; } msi->pcie_base = pcie->base; msi->data_address = pcie->msi_data_address; msi->irq = irq; msi->legacy = pcie->misc_revision < BRCM_PCIE_HW_REV_33; rt_spin_lock_init(&msi->lock); if (msi->legacy) { msi->intr_base = msi->pcie_base + PCIE_INTR2_CPU_BASE; msi->msi_max_nr = BRCM_INT_PCI_MSI_LEGACY_NR; msi->legacy_shift = 24; } else { msi->intr_base = msi->pcie_base + PCIE_MSI_INTR2_BASE; msi->msi_max_nr = 64; msi->legacy_shift = 0; } msi->inner_pic.priv_data = msi; msi->inner_pic.ops = &brcm_msi_ops; rt_pic_linear_irq(&msi->inner_pic, msi->msi_max_nr); rt_pic_user_extends(&msi->inner_pic); rt_hw_interrupt_install(msi->irq, brcm_pcie_msi_isr, msi, "brcm-pcie-msi"); rt_hw_interrupt_umask(msi->irq); val = msi->legacy ? BRCM_INT_PCI_MSI_LEGACY_MASK : BRCM_INT_PCI_MSI_MASK; HWREG32(msi->intr_base + MSI_INT_MASK_CLR) = val; HWREG32(msi->intr_base + MSI_INT_CLR) = val; /* * The 0 bit of PCIE_MISC_MSI_BAR_CONFIG_LO is repurposed to MSI enable, * which we set to 1. */ HWREG32(msi->pcie_base + PCIE_MISC_MSI_BAR_CONFIG_LO) = rt_lower_32_bits(msi->data_address) | 0x1; HWREG32(msi->pcie_base + PCIE_MISC_MSI_BAR_CONFIG_HI) = rt_upper_32_bits(msi->data_address); val = msi->legacy ? PCIE_MISC_MSI_DATA_CONFIG_VAL_8 : PCIE_MISC_MSI_DATA_CONFIG_VAL_32; HWREG32(msi->pcie_base + PCIE_MISC_MSI_DATA_CONFIG) = val; pcie->msi = msi; rt_ofw_data(dev->ofw_node) = &msi->inner_pic; rt_ofw_node_set_flag(dev->ofw_node, RT_OFW_F_READLY); return RT_EOK; } rt_inline rt_err_t brcm_phy_start(struct brcm_pcie *pcie) { return pcie->soc_data->has_phy ? brcm_phy_cntl(pcie, 1) : RT_EOK; } rt_inline rt_err_t brcm_phy_stop(struct brcm_pcie *pcie) { return pcie->soc_data->has_phy ? brcm_phy_cntl(pcie, 0) : RT_EOK; } static void brcm_pcie_turn_off(struct brcm_pcie *pcie) { rt_uint32_t tmp; void *base = pcie->base; if (brcm_pcie_link_up(pcie)) { brcm_pcie_enter_l23(pcie); } /* Assert fundamental reset */ pcie->soc_data->perst_set(pcie, 1); /* Deassert request for L23 in case it was asserted */ tmp = HWREG32(base + PCIE_MISC_PCIE_CTRL); replace_u32_bitfield(&tmp, 0, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK); HWREG32(base + PCIE_MISC_PCIE_CTRL) = tmp; if (pcie->soc_data->type != BCM2711) { /* Turn off SerDes */ tmp = HWREG32(base + HARD_DEBUG(pcie)); replace_u32_bitfield(&tmp, 1, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK); HWREG32(base + HARD_DEBUG(pcie)) = tmp; } if (!(pcie->soc_data->quirks & CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN)) { /* Shutdown PCIe bridge */ pcie->soc_data->bridge_sw_init_set(pcie, 1); } } static void add_inbound_win(struct inbound_win *b, rt_uint8_t *count, rt_uint64_t size, rt_uint64_t cpu_addr, rt_uint64_t pci_offset) { b->size = size; b->cpu_addr = cpu_addr; b->pci_offset = pci_offset; (*count)++; } /* * Debian DT omits the MIP doorbell dma-ranges entry present in bcm2712.dtsi. * EP MSIX MemWr to brcm,msi-pci-addr must inbound-map to the MIP MMIO block. */ static void brcm_pcie_add_mip_doorbell_win(struct brcm_pcie *pcie, struct inbound_win *b, rt_uint8_t *n, const struct inbound_win *begin) { struct rt_ofw_node *np, *msi_np; rt_uint32_t addr_cells[2]; rt_uint64_t pci_door, cpu_door, reg_size; int i; if (pcie->soc_data->type != BCM7712 || !pcie->dev) { return; } np = pcie->dev->ofw_node; msi_np = rt_ofw_parse_phandle(np, "msi-parent", 0); if (!msi_np || msi_np == np) { rt_ofw_node_put(msi_np); return; } if (!rt_ofw_node_is_compatible(msi_np, "brcm,bcm2712-mip-intc")) { rt_ofw_node_put(msi_np); return; } if (rt_ofw_prop_read_u32_array_index(msi_np, "brcm,msi-pci-addr", 0, 2, addr_cells) < 2 || rt_ofw_get_address(msi_np, 0, &cpu_door, ®_size)) { rt_ofw_node_put(msi_np); return; } pci_door = ((rt_uint64_t)addr_cells[0] << 32) | addr_cells[1]; for (i = 1; i <= *n; ++i) { if (begin[i].pci_offset == pci_door) { rt_ofw_node_put(msi_np); return; } } if (*n >= pcie->soc_data->num_inbound_wins) { LOG_W("No inbound slot for MIP doorbell pci=%#llx", (unsigned long long)pci_door); rt_ofw_node_put(msi_np); return; } add_inbound_win(b, n, 0x1000, cpu_door, pci_door); rt_ofw_node_put(msi_np); } static int brcm_pcie_get_inbound_wins(struct brcm_pcie *pcie, struct inbound_win inbound_wins[]) { rt_uint8_t n = 0; rt_uint64_t lowest_pcie_addr = ~(rt_uint64_t)0; rt_uint64_t pci_offset, cpu_addr, size = 0, tot_size = 0; struct rt_pci_host_bridge *bridge = pcie->bridge; /* * The HW registers (and PCIe) use order-1 numbering for BARs. As such, * we have inbound_wins[0] unused and BAR1 starts at inbound_wins[1]. */ struct inbound_win *b_begin = &inbound_wins[1], *b = b_begin; /* * STB chips beside 7712 disable the first inbound window default. * Rather being mapped to system memory it is mapped to the * internal registers of the SoC. This feature is deprecated, has * security considerations, and is not implemented in our modern * SoCs. */ if (pcie->soc_data->type != BCM7712) { add_inbound_win(b++, &n, 0, 0, 0); } for (int i = 0; i < bridge->dma_regions_nr; ++i) { rt_uint64_t pcie_start, cpu_start; struct rt_pci_bus_region *region = &bridge->dma_regions[i]; pcie_start = region->phy_addr; cpu_start = region->cpu_addr; size = region->size; tot_size += size; if (pcie_start < lowest_pcie_addr) { lowest_pcie_addr = pcie_start; } /* * 7712 and newer chips may have many BARs, with each * offering a non-overlapping viewport to system memory. * That being said, each BARs size must still be a power of two. */ if (pcie->soc_data->type == BCM7712) { add_inbound_win(b++, &n, size, cpu_start, pcie_start); } if (n > pcie->soc_data->num_inbound_wins) { break; } } if (lowest_pcie_addr == ~(rt_uint64_t)0) { return -RT_EINVAL; } /* * 7712 and newer chips do not have an internal memory mapping system * that enables multiple memory controllers. As such, it can return * now w/o doing special configuration. */ if (pcie->soc_data->type == BCM7712) { brcm_pcie_add_mip_doorbell_win(pcie, b, &n, inbound_wins); if (n > pcie->soc_data->num_inbound_wins) { LOG_E("Too many inbound windows (%u)", n); return -RT_EINVAL; } return n; } pcie->memc_nr = rt_dm_dev_prop_read_u64_array_index(pcie->dev, "brcm,scb-sizes", 0, PCIE_BRCM_MAX_MEMC, pcie->memc_size); if ((rt_ssize_t)pcie->memc_nr <= 0) { /* Make an educated guess */ pcie->memc_nr = 1; pcie->memc_size[0] = 1ULL << __fls64(tot_size - 1); } /* Each memc is viewed through a "port" that is a power of 2 */ for (int i = 0, size = 0; i < pcie->memc_nr; ++i) { size += pcie->memc_size[i]; } /* Our HW mandates that the window size must be a power of 2 */ size = 1ULL << __fls64(size - 1); /* * Pi4 firmware rewrites dma-ranges before boot. * Ignore bogus brcm,scb-sizes and derive the inbound aperture from the * parsed dma-ranges total instead. */ if (pcie->soc_data->type == BCM2711) { if (tot_size >= SIZE_MB) { rt_uint64_t want = 1ULL << __fls64(tot_size - 1); if (size < want) { size = want; } } else if (lowest_pcie_addr == 0x400000000ULL) { size = 2ULL * SIZE_GB; } } /* * For STB chips, the BAR2 cpu_addr is hardwired to the start * of system memory, so we set it to 0. */ cpu_addr = 0; pci_offset = lowest_pcie_addr; /* * We validate the inbound memory view even though we should trust * whatever the device-tree provides. This is because of an HW issue on * early Raspberry Pi 4's revisions (bcm2711). It turns out its * firmware has to dynamically edit dma-ranges due to a bug on the * PCIe controller integration, which prohibits any access above the * lower 3GB of memory. Given this, we decided to keep the dma-ranges * in check, avoiding hard to debug device-tree related issues in the * future: * * The PCIe host controller by design must set the inbound viewport to * be a contiguous arrangement of all of the system's memory. In * addition, its size mut be a power of two. To further complicate * matters, the viewport must start on a pcie-address that is aligned * on a multiple of its size. If a portion of the viewport does not * represent system memory -- e.g. 3GB of memory requires a 4GB * viewport -- we can map the outbound memory in or after 3GB and even * though the viewport will overlap the outbound memory the controller * will know to send outbound memory downstream and everything else * upstream. * * For example: * * - The best-case scenario, memory up to 3GB, is to place the inbound * region in the first 4GB of pcie-space, as some legacy devices can * only address 32bits. We would also like to put the MSI under 4GB * as well, since some devices require a 32bit MSI target address. * * - If the system memory is 4GB or larger we cannot start the inbound * region at location 0 (since we have to allow some space for * outbound memory @ 3GB). So instead it will start at the 1x * multiple of its size */ if (!size || (pci_offset & (size - 1)) || (pci_offset < 4UL * SIZE_GB && pci_offset > 2UL * SIZE_GB)) { return -RT_EINVAL; } /* Enable inbound window 2, the main inbound window for STB chips */ add_inbound_win(b++, &n, size, cpu_addr, pci_offset); /* * Disable inbound window 3. On some chips presents the same * window as #2 but the data appears in a settable endianness. */ add_inbound_win(b++, &n, 0, 0, 0); return n; } static rt_uint32_t brcm_bar_reg_offset(int bar) { if (bar <= 3) { return PCIE_MISC_RC_BAR1_CONFIG_LO + 8 * (bar - 1); } else { return PCIE_MISC_RC_BAR4_CONFIG_LO + 8 * (bar - 4); } } static rt_uint32_t brcm_ubus_reg_offset(int bar) { if (bar <= 3) { return PCIE_MISC_UBUS_BAR1_CONFIG_REMAP + 8 * (bar - 1); } else { return PCIE_MISC_UBUS_BAR4_CONFIG_REMAP + 8 * (bar - 4); } } static void set_inbound_win_registers(struct brcm_pcie *pcie, const struct inbound_win *inbound_wins, rt_uint8_t num_inbound_wins) { void *base = pcie->base; for (int i = 1; i <= num_inbound_wins; ++i) { rt_uint64_t pci_offset = inbound_wins[i].pci_offset; rt_uint64_t cpu_addr = inbound_wins[i].cpu_addr; rt_uint64_t size = inbound_wins[i].size; rt_uint32_t reg_offset = brcm_bar_reg_offset(i); rt_uint32_t tmp = rt_lower_32_bits(pci_offset); replace_u32_bitfield(&tmp, brcm_pcie_encode_ibar_size(size), PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK); /* Write low */ HWREG32(base + reg_offset) = tmp; /* Write high */ HWREG32(base + reg_offset + 4) = rt_upper_32_bits(pci_offset); /* * Most STB chips: * Do nothing. * 7712: * All of their BARs need to be set. */ if (pcie->soc_data->type == BCM7712) { /* BUS remap register settings */ reg_offset = brcm_ubus_reg_offset(i); tmp = rt_lower_32_bits(cpu_addr) & ~0xfffU; tmp |= PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_ENABLE_MASK; HWREG32(base + reg_offset) = tmp; HWREG32(base + reg_offset + 4) = rt_upper_32_bits(cpu_addr); } } } static void brcm_pcie_reapply_inbound_access(struct brcm_pcie *pcie) { void *base = pcie->base; if (!base || pcie->soc_data->type != BCM7712) { return; } brcm_pcie_misc_ctrl_enable_access(pcie); /* * MIP MSIX doorbell lands on inbound BAR3 (PCI ff_ffff_f000), not BAR2. * Re-enable ACCESS on every active inbound UBUS remap after link events. */ for (int i = 1; i <= pcie->num_inbound_wins; ++i) { rt_uint32_t reg = brcm_ubus_reg_offset(i); rt_uint32_t tmp; if (pcie->inbound_wins[i].size == 0) { continue; } tmp = HWREG32(base + reg); replace_u32_bitfield(&tmp, 1, PCIE_MISC_UBUS_BAR2_CONFIG_REMAP_ACCESS_ENABLE_MASK); HWREG32(base + reg) = tmp; } } static void brcm_pcie_reapply_7712_maps(struct brcm_pcie *pcie) { if (pcie->soc_data->type != BCM7712 || !pcie->num_inbound_wins) { return; } brcm_pcie_reapply_inbound_access(pcie); set_inbound_win_registers(pcie, pcie->inbound_wins, pcie->num_inbound_wins); brcm_pcie_apply_outbound_wins(pcie); } static rt_err_t brcm_pcie_setup(struct brcm_pcie *pcie) { rt_err_t err; void *base = pcie->base; int num_out_wins = 0, num_inbound_wins = 0; rt_uint32_t tmp, num_lanes = 0, num_lanes_cap = 0; struct rt_pci_host_bridge *bridge = pcie->bridge; struct inbound_win inbound_wins[PCIE_BRCM_MAX_INBOUND_WINS]; /* Reset the bridge */ pcie->soc_data->bridge_sw_init_set(pcie, 1); /* Ensure that PERST# is asserted; some bootloaders may deassert it. */ if (pcie->soc_data->type == BCM2711) { if ((err = pcie->soc_data->perst_set(pcie, 1))) { pcie->soc_data->bridge_sw_init_set(pcie, 0); return err; } } rt_hw_us_delay(150); /* Take the bridge out of reset */ if ((err = pcie->soc_data->bridge_sw_init_set(pcie, 0))) { return err; } /* * On some BCM2711 integrations this register window can raise SError * immediately after bridge reset deassert; skip this write there. */ if (pcie->soc_data->type != BCM2711) { tmp = HWREG32(base + HARD_DEBUG(pcie)); if (is_bmips(pcie)) { tmp &= ~PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK; } else { tmp &= ~PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK; } HWREG32(base + HARD_DEBUG(pcie)) = tmp; /* Wait for SerDes to be stable */ rt_hw_us_delay(150); } brcm_pcie_misc_ctrl_enable_access(pcie); num_inbound_wins = brcm_pcie_get_inbound_wins(pcie, inbound_wins); if (num_inbound_wins < 0) { return num_inbound_wins; } set_inbound_win_registers(pcie, inbound_wins, num_inbound_wins); pcie->num_inbound_wins = (rt_uint8_t)num_inbound_wins; rt_memcpy(pcie->inbound_wins, inbound_wins, sizeof(inbound_wins)); if (!brcm_pcie_rc_mode(pcie)) { LOG_E("PCIe RC controller misconfigured as Endpoint"); return -RT_EINVAL; } tmp = HWREG32(base + PCIE_MISC_MISC_CTRL); for (int memc = 0; memc < pcie->memc_nr; memc++) { rt_uint32_t scb_size_val = rt_ilog2(pcie->memc_size[memc]) - 15; if (memc == 0) { replace_u32_bitfield(&tmp, scb_size_val, SCB_SIZE_MASK(0)); } else if (memc == 1) { replace_u32_bitfield(&tmp, scb_size_val, SCB_SIZE_MASK(1)); } else if (memc == 2) { replace_u32_bitfield(&tmp, scb_size_val, SCB_SIZE_MASK(2)); } } HWREG32(base + PCIE_MISC_MISC_CTRL) = tmp; /* * We ideally want the MSI target address to be located in the 32bit * addressable memory area. Some devices might depend on it. This is * possible either when the inbound window is located above the lower * 4GB or when the inbound area is smaller than 4GB (taking into * account the rounding-up we're forced to perform). */ if (inbound_wins[2].pci_offset >= (4UL * SIZE_GB) || (inbound_wins[2].size + inbound_wins[2].pci_offset) < (4UL * SIZE_GB)) { pcie->msi_data_address = BRCM_MSI_TARGET_ADDR_LT_4GB; } else { pcie->msi_data_address = BRCM_MSI_TARGET_ADDR_GT_4GB; } /* Don't advertise L0s capability if 'aspm-no-l0s' */ tmp = HWREG32(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); if (rt_dm_dev_prop_read_bool(pcie->dev, "aspm-no-l0s")) { tmp &= ~PCI_EXP_LNKCAP_ASPM_L0S; } HWREG32(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY) = tmp; num_lanes_cap = (tmp & PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_MAX_LINK_WIDTH_MASK) >> 4; /* * Use hardware negotiated Max Link Width value by default. If the * "num-lanes" DT property is present, assume that the chip's default * link width capability information is incorrect/undesired and use the * specified value instead. */ if (!rt_dm_dev_prop_read_u32(pcie->dev, "num-lanes", &num_lanes) && num_lanes && num_lanes <= 4 && num_lanes_cap != num_lanes) { replace_u32_bitfield(&tmp, num_lanes, PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_MAX_LINK_WIDTH_MASK); HWREG32(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY) = tmp; tmp = HWREG32(base + PCIE_RC_PL_REG_PHY_CTL_1); replace_u32_bitfield(&tmp, 1, PCIE_RC_PL_REG_PHY_CTL_1_REG_P2_POWERDOWN_ENA_NOSYNC_MASK); HWREG32(base + PCIE_RC_PL_REG_PHY_CTL_1) = tmp; } /* * For config space accesses on the RC, show the right class for * a PCIe-PCIe bridge (the default setting is to be EP mode). */ tmp = HWREG32(base + PCIE_RC_CFG_PRIV1_ID_VAL3); replace_u32_bitfield(&tmp, 0x060400, PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK); HWREG32(base + PCIE_RC_CFG_PRIV1_ID_VAL3) = tmp; for (int i = 0; i < bridge->bus_regions_nr; ++i) { struct rt_pci_bus_region *region = &bridge->bus_regions[i]; if (region->flags != PCI_BUS_REGION_F_MEM && region->flags != PCI_BUS_REGION_F_PREFETCH) { continue; } if (num_out_wins >= BRCM_NUM_PCIE_OUT_WINS) { LOG_E("Too many outbound wins"); return -RT_EINVAL; } if (is_bmips(pcie)) { rt_uint64_t start = region->cpu_addr; rt_off_t offset = region->cpu_addr - region->phy_addr; rt_uint32_t nwins = region->size / 128 * SIZE_MB; /* BMIPS PCIe outbound windows have a 128MB max size */ if (nwins > BRCM_NUM_PCIE_OUT_WINS) { nwins = BRCM_NUM_PCIE_OUT_WINS; } for (int j = 0; j < nwins; ++j, start += 128 * SIZE_MB) { brcm_pcie_set_outbound_win(pcie, j, start, start - offset, 128 * SIZE_MB); } break; } brcm_pcie_set_outbound_win(pcie, num_out_wins, region->cpu_addr, region->phy_addr, region->size); ++num_out_wins; } /* PCIe->SCB endian mode for BAR */ tmp = HWREG32(base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1); replace_u32_bitfield(&tmp, PCIE_RC_CFG_VENDOR_SPECIFIC_REG1_LITTLE_ENDIAN, PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK); HWREG32(base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1) = tmp; if (pcie->soc_data->post_setup) { if ((err = pcie->soc_data->post_setup(pcie))) { return err; } } return RT_EOK; } static void brcm_pcie_free(struct brcm_pcie *pcie) { if (pcie->bridge) { rt_pci_host_bridge_remove(pcie->bridge); } if (pcie->soc_data && pcie->soc_data->has_phy) { brcm_phy_stop(pcie); } brcm_pcie_turn_off(pcie); if (!rt_is_err_or_null(pcie->rescal_rstc)) { rt_reset_control_assert(pcie->rescal_rstc); rt_reset_control_put(pcie->rescal_rstc); } if (!rt_is_err_or_null(pcie->perst_rstc)) { rt_reset_control_put(pcie->perst_rstc); } if (!rt_is_err_or_null(pcie->bridge_rstc)) { rt_reset_control_put(pcie->bridge_rstc); } if (!rt_is_err_or_null(pcie->swinit_rstc)) { rt_reset_control_put(pcie->swinit_rstc); } if (!rt_is_err_or_null(pcie->clk)) { rt_clk_disable_unprepare(pcie->clk); } if (pcie->base) { rt_iounmap(pcie->base); } rt_pci_host_bridge_free(pcie->bridge); } static rt_err_t brcm_pcie_get_optional_reset(struct rt_device *dev, const char *name, struct rt_reset_control **rstc) { struct rt_reset_control *rc = rt_reset_control_get_by_name(dev, name); if (rt_is_err(rc)) { if (rt_ptr_err(rc) == -RT_ENOENT) { *rstc = RT_NULL; return RT_EOK; } return rt_ptr_err(rc); } *rstc = rc; return RT_EOK; } static rt_err_t brcm_pcie_probe(struct rt_platform_device *pdev) { rt_err_t err; struct brcm_pcie *pcie; struct rt_device *dev = &pdev->parent; struct rt_pci_host_bridge *bridge; struct rt_ofw_node *np = dev->ofw_node, *msi_np; bridge = rt_pci_host_bridge_alloc(sizeof(*pcie)); if (!bridge) { return -RT_ENOMEM; } pcie = (void *)bridge->priv; pcie->soc_data = pdev->id->data; pcie->dev = dev; pcie->bridge = bridge; pcie->base = rt_dm_dev_iomap(dev, 0); if (!pcie->base) { err = -RT_EIO; goto _fail; } pcie->clk = rt_clk_get_by_name(dev, "sw_pcie"); if (rt_is_err(pcie->clk)) { if (rt_ptr_err(pcie->clk) != -RT_ENOENT) { err = rt_ptr_err(pcie->clk); goto _fail; } pcie->clk = RT_NULL; } else if ((err = rt_clk_prepare_enable(pcie->clk))) { LOG_E("%s: could not enable clock", rt_dm_dev_get_name(dev)); goto _fail; } if (rt_dm_dev_prop_read_u32(dev, "max-link-speed", &pcie->generation)) { pcie->generation = 0; } pcie->ssc = rt_dm_dev_prop_read_bool(dev, "brcm,enable-ssc"); rt_dm_dev_prop_read_u32(dev, "brcm,tperst-clk-ms", &pcie->tperst_clk_ms); if ((err = brcm_pcie_get_optional_reset(dev, "rescal", &pcie->rescal_rstc))) { goto _fail; } if ((err = brcm_pcie_get_optional_reset(dev, "perst", &pcie->perst_rstc))) { goto _fail; } if ((err = brcm_pcie_get_optional_reset(dev, "bridge", &pcie->bridge_rstc))) { goto _fail; } if ((err = brcm_pcie_get_optional_reset(dev, "swinit", &pcie->swinit_rstc))) { goto _fail; } pcie->soc_data->bridge_sw_init_set(pcie, 0); if (pcie->swinit_rstc) { if ((err = rt_reset_control_assert(pcie->swinit_rstc))) { goto _fail; } rt_hw_us_delay(1); if ((err = rt_reset_control_deassert(pcie->swinit_rstc))) { goto _fail; } } if (pcie->rescal_rstc && (err = rt_reset_control_reset(pcie->rescal_rstc))) { goto _fail; } if ((err = brcm_phy_start(pcie))) { goto _fail; } bridge->parent.ofw_node = np; if ((err = rt_pci_host_bridge_init(bridge))) { goto _fail; } if ((err = brcm_pcie_setup(pcie))) { goto _fail; } pcie->misc_revision = HWREG32(pcie->base + PCIE_MISC_REVISION); if (pcie->soc_data->type == BCM4908 && pcie->misc_revision >= BRCM_PCIE_HW_REV_3_20) { LOG_E("%s: hardware revision with unsupported PERST# setup", rt_dm_dev_get_name(dev)); err = -RT_EINVAL; goto _fail; } msi_np = rt_ofw_parse_phandle(np, "msi-parent", 0); if (msi_np == np) { if ((err = brcm_pcie_enable_msi(pcie))) { LOG_E("%s: probe ofw internal MSI error = %s", rt_dm_dev_get_name(dev), rt_strerror(err)); goto _fail; } } rt_ofw_node_put(msi_np); bridge->ops = pcie->soc_data->type == BCM7425 ? &brcm7425_pcie_ops : &brcm_pcie_ops; bridge->sysdata = pcie; if ((err = rt_pci_host_bridge_probe(bridge))) { goto _fail; } if (!brcm_pcie_link_up(pcie)) { err = -RT_EINVAL; goto _fail; } brcm_pcie_reapply_7712_maps(pcie); dev->user_data = pcie; return RT_EOK; _fail: if (pcie->msi) { rt_pic_cancel_irq(&pcie->msi->inner_pic); } brcm_pcie_free(pcie); return err; } static rt_err_t brcm_pcie_remove(struct rt_platform_device *pdev) { struct brcm_pcie *pcie = pdev->parent.user_data; brcm_pcie_free(pcie); return RT_EOK; } static const struct rt_ofw_node_id brcm_pcie_ofw_ids[] = { { .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg }, { .compatible = "brcm,bcm2712-pcie", .data = &bcm2712_cfg }, { .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg }, { .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg }, { .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg }, { .compatible = "brcm,bcm7216-pcie", .data = &bcm7216_cfg }, { .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg }, { .compatible = "brcm,bcm7435-pcie", .data = &bcm7435_cfg }, { .compatible = "brcm,bcm7425-pcie", .data = &bcm7425_cfg }, { .compatible = "brcm,bcm7712-pcie", .data = &bcm7712_cfg }, { /* sentinel */ } }; void brcm_pcie_reapply_maps(struct rt_pci_device *pdev) { struct brcm_pcie *pcie; struct rt_pci_host_bridge *host_bridge; if (!pdev || !pdev->bus) { return; } if (!(host_bridge = rt_pci_find_host_bridge(pdev->bus))) { return; } if ((pcie = host_bridge->sysdata)) { brcm_pcie_reapply_7712_maps(pcie); } } static struct rt_platform_driver brcm_pcie_driver = { .name = "pcie-brcm", .ids = brcm_pcie_ofw_ids, .probe = brcm_pcie_probe, .remove = brcm_pcie_remove, }; RT_PLATFORM_DRIVER_EXPORT(brcm_pcie_driver);