From c856eff2d817028992484917cbedb8db2817099f Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Mon, 9 Nov 2026 14:45:02 -0500 Subject: [PATCH] PCI: apple: restore the J713 native radio port after system sleep Rebuild the native port, GPIOs, requester/MSI routes and DART before PCI resumes the BCM4388 functions. Keep endpoint bus mastering disabled until the translation context is restored and the link and identities agree. Require both bound endpoint drivers to participate in the sleep transaction. A refused sleep restores the retained link without resetting it; a failed power-loss recovery stops before other DMA users resume. Enable the J713 radio and publish its CLKREQ GPIO with the host support. Add host lifecycle and register-failure fixtures. Signed-off-by: Erik Bourget --- MAINTAINERS | 3 + arch/arm64/boot/dts/apple/t8132-j713.dts | 10 +- drivers/pci/controller/pcie-apple-radio.c | 845 ++++++++++++++++++++++ drivers/pci/controller/pcie-apple.c | 55 +- include/linux/soc/apple/radio-sleep.h | 45 ++ tools/asahi/radio-pm-tests.c | 313 ++++++++ tools/asahi/radio-pm-tests.py | 43 ++ tools/asahi/radio-sleep-fence-tests.c | 98 +++ tools/asahi/radio-sleep-fence-tests.py | 33 + tools/asahi/radio-sleep-power-tests.c | 413 +++++++++++ tools/asahi/radio-sleep-power-tests.py | 49 ++ tools/asahi/radio-sleep-snapshot-tests.c | 133 ++++ tools/asahi/radio-sleep-snapshot-tests.py | 76 ++ 13 files changed, 2105 insertions(+), 11 deletions(-) create mode 100644 drivers/pci/controller/pcie-apple-radio.c create mode 100644 include/linux/soc/apple/radio-sleep.h create mode 100644 tools/asahi/radio-pm-tests.c create mode 100644 tools/asahi/radio-pm-tests.py create mode 100644 tools/asahi/radio-sleep-fence-tests.c create mode 100644 tools/asahi/radio-sleep-fence-tests.py create mode 100644 tools/asahi/radio-sleep-power-tests.c create mode 100644 tools/asahi/radio-sleep-power-tests.py create mode 100644 tools/asahi/radio-sleep-snapshot-tests.c create mode 100644 tools/asahi/radio-sleep-snapshot-tests.py diff --git a/MAINTAINERS b/MAINTAINERS index 7d1c382eff471..0a06b6c9eee48 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2068,7 +2068,10 @@ APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier L: linux-pci@vger.kernel.org S: Maintained +F: drivers/pci/controller/pcie-apple-radio.c F: drivers/pci/controller/pcie-apple.c +F: include/linux/soc/apple/radio-sleep.h +F: tools/asahi/radio-*-tests.* APPLE SMC DRIVER M: Henrik Rydberg diff --git a/arch/arm64/boot/dts/apple/t8132-j713.dts b/arch/arm64/boot/dts/apple/t8132-j713.dts index a751a81ac8bf6..3758e4a4429ad 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713.dts +++ b/arch/arm64/boot/dts/apple/t8132-j713.dts @@ -167,11 +167,15 @@ apple,pmc-interface = <&pmc_disp0>; }; -/* Bring-up: nothing brings the radio's PCIe port back after a deep sleep yet. */ +/* Native radio recovery is required before admitting system sleep. */ &pcie0 { - status = "disabled"; + status = "okay"; }; &dart_apcie0 { - status = "disabled"; + status = "okay"; +}; + +&port00 { + clkreq-gpios = <&pinctrl_ap 160 GPIO_ACTIVE_LOW>; }; diff --git a/drivers/pci/controller/pcie-apple-radio.c b/drivers/pci/controller/pcie-apple-radio.c new file mode 100644 index 0000000000000..e6eb8a03d4eaf --- /dev/null +++ b/drivers/pci/controller/pcie-apple-radio.c @@ -0,0 +1,845 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by pcie-apple.c. J713 BCM4388 firmware retention and host recovery. */ + +#define RADIO_PHY_RESET BIT(4) +#define RADIO_PORT_TIMEOUT_US 50000 +#define RADIO_DBI_RO_WR 0x8bc +#define RADIO_DBI_LINK_CONTROL 0x710 +#define RADIO_DBI_WIDTH_SPEED 0x80c + +enum apple_radio_phase { + APPLE_RADIO_IDLE, + APPLE_RADIO_OPEN, + APPLE_RADIO_SEALING, + APPLE_RADIO_SEALED, + APPLE_RADIO_VERIFIED, + APPLE_RADIO_ABORTED, + APPLE_RADIO_FAILED, + APPLE_RADIO_STOPPING, + APPLE_RADIO_STOPPED, + APPLE_RADIO_RECOVERING, +}; + +struct apple_radio_function { + struct pci_dev *pdev; + u16 command; + bool attempted, held; + u64 suspended, resumed; + int error; + void (*finish)(struct device *dev); +}; + +struct apple_radio_sleep { + struct apple_pcie *pcie; + struct apple_pcie_port *port; + struct apple_dart_sleep dart; + struct apple_radio_function fn[2]; + struct task_struct *owner; + u64 generation; + enum apple_radio_phase phase; + bool bound, host_suspended, snapshot; + bool recovery_ready, config_ready; + struct gpio_desc *perst, *clkreq; + struct pci_dev *root; + struct apple_tunable *tables[4]; + u32 intmask, appclk; + u32 rid[16], msi[32], msicfg, msibase, msiaddr, msiaddr_hi; + DECLARE_BITMAP(vectors, 32); + DECLARE_BITMAP(sids, 16); +}; + +/* A single exact board/controller may own this publication during system PM. */ +static struct apple_radio_sleep *apple_radio_active; + +static void __noreturn apple_radio_terminal(void) +{ + panic("J713 radio recovery failed; DMA remains blocked"); +} + +static bool apple_radio_writer_closed(struct apple_pcie *pcie) +{ + return READ_ONCE(pcie->radio_closed); +} + +static bool apple_radio_config_allowed(struct apple_pcie *pcie) +{ + struct apple_radio_sleep *s = pcie->radio; + + if (!s || READ_ONCE(apple_radio_active) != s) + return true; + /* Pair with publication after the complete port/DART/link recovery. */ + switch (smp_load_acquire(&s->phase)) { + case APPLE_RADIO_STOPPING: + case APPLE_RADIO_STOPPED: + case APPLE_RADIO_FAILED: + return false; + case APPLE_RADIO_RECOVERING: + return current == s->owner && READ_ONCE(s->config_ready); + default: + return true; + } +} + +static int apple_radio_bme_hold(struct apple_radio_function *fn) +{ + u16 value; + + if (pci_read_config_word(fn->pdev, PCI_COMMAND, &value) || value == 0xffff) + return -EIO; + fn->command = value; + /* A failed config write may already have reached the function. */ + fn->attempted = true; + if (pci_write_config_word(fn->pdev, PCI_COMMAND, value & ~PCI_COMMAND_MASTER) || + pci_read_config_word(fn->pdev, PCI_COMMAND, &value) || + value != (fn->command & ~PCI_COMMAND_MASTER)) + return -EIO; + fn->held = true; + return 0; +} + +static int apple_radio_bme_restore(struct apple_radio_function *fn) +{ + u16 value; + + if (!fn->attempted) + return 0; + if (pci_read_config_word(fn->pdev, PCI_COMMAND, &value) || value == 0xffff) + return -EUCLEAN; + /* Restore only our own change, never overwrite a foreign config mutation. */ + if (value != fn->command && value != (fn->command & ~PCI_COMMAND_MASTER)) + return -EUCLEAN; + if (value != fn->command && pci_write_config_word(fn->pdev, PCI_COMMAND, fn->command)) + return -EUCLEAN; + if (pci_read_config_word(fn->pdev, PCI_COMMAND, &value) || value != fn->command) + return -EUCLEAN; + fn->held = false; + fn->attempted = false; + return 0; +} + +static struct apple_radio_function *apple_radio_endpoint(struct device *dev) +{ + /* Acquire the initialized generation published by begin. */ + struct apple_radio_sleep *s = smp_load_acquire(&apple_radio_active); + int i; + + if (!s) + return NULL; + for (i = 0; i < ARRAY_SIZE(s->fn); i++) + if (&s->fn[i].pdev->dev == dev) + return &s->fn[i]; + return NULL; +} + +bool apple_radio_sleep_owned(struct device *dev) +{ + return apple_radio_endpoint(dev) != NULL; +} +EXPORT_SYMBOL_GPL(apple_radio_sleep_owned); + +int apple_radio_sleep_endpoint_prepare(struct device *dev, void (*finish)(struct device *dev)) +{ + struct apple_radio_function *fn = apple_radio_endpoint(dev); + struct apple_radio_sleep *s; + + if (!fn) + return 0; + s = READ_ONCE(apple_radio_active); + if (READ_ONCE(s->phase) != APPLE_RADIO_OPEN || pm_suspend_target_state != PM_SUSPEND_MEM) + return -EPERM; + fn->finish = finish; + return 0; +} +EXPORT_SYMBOL_GPL(apple_radio_sleep_endpoint_prepare); + +int apple_radio_sleep_endpoint_suspended(struct device *dev) +{ + struct apple_radio_function *fn = apple_radio_endpoint(dev); + struct apple_radio_sleep *s; + + if (!fn) + return 0; + s = READ_ONCE(apple_radio_active); + if (READ_ONCE(s->phase) != APPLE_RADIO_OPEN || fn->suspended) + return -ESTALE; + /* Publish the endpoint stop before seal examines the receipt. */ + smp_store_release(&fn->suspended, s->generation); + return 0; +} +EXPORT_SYMBOL_GPL(apple_radio_sleep_endpoint_suspended); + +int apple_radio_sleep_endpoint_resume(struct device *dev) +{ + struct apple_radio_function *fn = apple_radio_endpoint(dev); + struct apple_radio_sleep *s; + + if (!fn) + return 0; + s = READ_ONCE(apple_radio_active); + switch (READ_ONCE(s->phase)) { + case APPLE_RADIO_OPEN: /* Device PM may undo a failed late/noirq phase. */ + case APPLE_RADIO_ABORTED: + case APPLE_RADIO_VERIFIED: + return 0; + default: + /* Returning an errno would still let PM resume other DMA consumers. */ + apple_radio_terminal(); + } +} +EXPORT_SYMBOL_GPL(apple_radio_sleep_endpoint_resume); + +void apple_radio_sleep_endpoint_resumed(struct device *dev, int result) +{ + struct apple_radio_function *fn = apple_radio_endpoint(dev); + struct apple_radio_sleep *s; + + if (!fn) + return; + s = READ_ONCE(apple_radio_active); + WRITE_ONCE(fn->error, result); + /* Final finish must observe the endpoint result with its receipt. */ + smp_store_release(&fn->resumed, s->generation); +} +EXPORT_SYMBOL_GPL(apple_radio_sleep_endpoint_resumed); + +static int apple_radio_valid(struct apple_radio_sleep *s, u64 generation) +{ + if (!s->bound || !generation || s->generation != generation || s->owner != current || + READ_ONCE(apple_radio_active) != s) + return -ESTALE; + return s->phase == APPLE_RADIO_FAILED ? -EUCLEAN : 0; +} + +static int apple_radio_snapshot_error(struct apple_radio_sleep *s, const char *field, + unsigned int index, u32 value, u32 expected) +{ + /* Leave diagnostics silent if a caller has disabled interrupts. */ + if (!irqs_disabled()) + dev_err(s->pcie->dev, "radio sleep %s[%u]: observed=%#x expected=%#x\n", field, + index, value, expected); + return -EUCLEAN; +} + +static int apple_radio_snapshot(struct apple_radio_sleep *s, bool verify) +{ + struct apple_pcie *pcie = s->pcie; + struct apple_pcie_port *port = s->port; + u32 value, expected; + unsigned int i; + + value = apple_pcie_port_readl(port, PORT_STATUS); + if (!(value & PORT_STATUS_READY)) + return apple_radio_snapshot_error(s, "port-ready", 0, value, PORT_STATUS_READY); + value = apple_pcie_port_readl(port, PORT_LINKSTS); + /* + * This is an operating link, not the stopped PCIe-C port checked by + * tunnel initialization. Bit 2 can remain set while link-up is valid. + * DMA quiescence is established separately by the endpoint receipts, + * checked BME fences and pending-transaction waits below. + */ + if (!(value & PORT_LINKSTS_UP)) + return apple_radio_snapshot_error(s, "link-state", 0, value, PORT_LINKSTS_UP); + value = apple_pcie_port_readl(port, PORT_INTMSK); + if (verify && value != s->intmask) + return apple_radio_snapshot_error(s, "interrupt-mask", 0, value, s->intmask); + if (!verify) + s->intmask = value; + if (!verify) + s->appclk = apple_pcie_port_readl(port, PORT_APPCLK); + for (i = 0; i < ARRAY_SIZE(s->rid); i++) { + value = apple_pcie_port_readl(port, port_rid2sid_offset(port, i)); + if (verify && value != s->rid[i]) + return apple_radio_snapshot_error(s, "rid-compare", i, value, s->rid[i]); + if (!verify) + s->rid[i] = value; + } + for (i = 0; i < ARRAY_SIZE(s->fn); i++) { + expected = 0; + if (apple_pcie_managed_word(pcie->managed_sids, 0x100 + i, 1 + i, &expected) || + s->rid[1 + i] != expected) + return apple_radio_snapshot_error(s, "rid-admission", 1 + i, s->rid[1 + i], + expected); + } + for (i = 0; i < pcie->nvecs; i++) { + value = apple_pcie_port_readl(port, pcie->hw->port_msimap + 4 * i); + if (verify && value != s->msi[i]) + return apple_radio_snapshot_error(s, "msi-map", i, value, s->msi[i]); + if (!verify) + s->msi[i] = value; + } + value = apple_pcie_port_readl(port, PORT_MSICFG); + if (verify && value != s->msicfg) + return apple_radio_snapshot_error(s, "msi-config", 0, value, s->msicfg); + if (!verify) + s->msicfg = value; + value = apple_pcie_port_readl(port, PORT_MSIBASE); + if (verify && value != s->msibase) + return apple_radio_snapshot_error(s, "msi-base", 0, value, s->msibase); + if (!verify) + s->msibase = value; + value = apple_pcie_port_readl(port, pcie->hw->port_msiaddr); + if (verify && value != s->msiaddr) + return apple_radio_snapshot_error(s, "msi-address", 0, value, s->msiaddr); + if (!verify) + s->msiaddr = value; + value = apple_pcie_port_readl(port, pcie->hw->port_msiaddr_hi); + if (verify && value != s->msiaddr_hi) + return apple_radio_snapshot_error(s, "msi-address-high", 0, value, s->msiaddr_hi); + if (!verify) + s->msiaddr_hi = value; + if (!(s->msicfg & PORT_MSICFG_EN)) + return apple_radio_snapshot_error(s, "msi-enabled", 0, s->msicfg, PORT_MSICFG_EN); + if (verify) { + if (!bitmap_equal(s->vectors, pcie->bitmap, pcie->nvecs) || + !bitmap_equal(s->sids, port->sid_map, port->sid_map_sz)) + return -EUCLEAN; + } else { + bitmap_copy(s->vectors, pcie->bitmap, pcie->nvecs); + bitmap_copy(s->sids, port->sid_map, port->sid_map_sz); + } + return 0; +} + +static int apple_radio_commands_held(struct apple_radio_sleep *s) +{ + u16 value; + int i; + + for (i = 0; i < ARRAY_SIZE(s->fn); i++) + if (!s->fn[i].held || pci_read_config_word(s->fn[i].pdev, PCI_COMMAND, &value) || + value != (s->fn[i].command & ~PCI_COMMAND_MASTER)) + return -EUCLEAN; + return 0; +} + +static int apple_radio_begin(struct apple_radio_sleep *s, u64 generation) +{ + int i, ret; + + if (!s->bound || !generation || READ_ONCE(apple_radio_active) || + s->phase != APPLE_RADIO_IDLE || irqs_disabled() || in_interrupt() || + pm_suspend_target_state != PM_SUSPEND_MEM) + return -EBUSY; + /* QUIESCE is the only admitted BT path; do not silently change wake policy. */ + if (device_may_wakeup(&s->fn[1].pdev->dev)) + return -EBUSY; + if (!s->recovery_ready) + return -EOPNOTSUPP; + dev_info(s->pcie->dev, "radio sleep begin: link=%#x\n", + apple_pcie_port_readl(s->port, PORT_LINKSTS)); + ret = apple_dart_sleep_begin(&s->dart, generation); + if (ret) + return ret; + for (i = 0; i < ARRAY_SIZE(s->fn); i++) { + s->fn[i].attempted = s->fn[i].held = false; + s->fn[i].suspended = s->fn[i].resumed = 0; + s->fn[i].finish = NULL; + s->fn[i].error = 0; + } + s->snapshot = s->host_suspended = false; + s->config_ready = false; + s->owner = current; + s->generation = generation; + s->phase = APPLE_RADIO_OPEN; + /* Publish fully initialized endpoint and transaction records. */ + smp_store_release(&apple_radio_active, s); + return 0; +} + +static int apple_radio_seal(struct apple_radio_sleep *s, u64 generation) +{ + int i, ret = apple_radio_valid(s, generation); + + if (ret) + return ret; + if (s->phase != APPLE_RADIO_OPEN || irqs_disabled() || !s->host_suspended) + return -EPERM; + for (i = 0; i < ARRAY_SIZE(s->fn); i++) { + /* Pair with the endpoint suspend receipt publication. */ + if (smp_load_acquire(&s->fn[i].suspended) != generation) + return -EBUSY; + } + mutex_lock(&s->pcie->lock); + if (s->pcie->radio_writers || s->pcie->radio_closed) { + mutex_unlock(&s->pcie->lock); + return -EBUSY; + } + s->pcie->radio_closed = true; + mutex_unlock(&s->pcie->lock); + s->phase = APPLE_RADIO_SEALING; + /* Establish the host baseline before making even reversible BME changes. */ + ret = apple_radio_snapshot(s, false); + if (ret) { + dev_err(s->pcie->dev, "radio sleep host snapshot failed: %d\n", ret); + return ret; + } + s->snapshot = true; + for (i = 0; i < ARRAY_SIZE(s->fn); i++) { + ret = apple_radio_bme_hold(&s->fn[i]); + if (ret) + return ret; + if (!pci_wait_for_pending_transaction(s->fn[i].pdev)) + return -EBUSY; + } + ret = apple_dart_sleep_seal(&s->dart); + if (ret) { + dev_err(s->pcie->dev, "radio sleep DART seal failed: %d\n", ret); + return ret; + } + ret = apple_radio_snapshot(s, true) ?: apple_radio_commands_held(s); + if (ret) + return ret; + s->phase = APPLE_RADIO_SEALED; + return 0; +} + +static int apple_radio_port_restore(struct apple_radio_sleep *s) +{ + static const struct { + u32 offset, value; + } defaults[] = { + { 0x88, 0x110 }, { 0x100, ~0U }, { 0x148, ~0U }, { 0x210, ~0U }, + { 0x80, 0 }, { 0x84, 0 }, { 0x104, ~0U }, { 0x124, 0x100 }, + { 0x16c, 0 }, { 0x13c, 0x10 }, { 0x800, 0x100100 }, { 0x808, 0x1000ff }, + { 0x82c, 0 }, { 0x130, 0x3000000 }, { 0x140, 0x10 }, { 0x144, 0x253770 }, + { 0x21c, 0 }, { 0x834, 0 }, { 0x83c, 0 }, + }; + struct apple_pcie_port *port = s->port; + void __iomem *dbi = s->pcie->cfg->win; + u32 value; + int ret, speed = of_pci_get_max_link_speed(port->np); + + for (unsigned int i = 0; i < ARRAY_SIZE(defaults); i++) + apple_pcie_port_writel(port, defaults[i].value, defaults[i].offset); + apple_tunable_apply(port->base, s->tables[0]); + apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); + writel(readl(port->phy) & ~(PHY_LANE_CFG_REFCLK0REQ | PHY_LANE_CFG_REFCLK1REQ), port->phy); + writel(readl(port->phy) | PHY_LANE_CFG_REFCLK0REQ, port->phy); + ret = readl_poll_timeout(port->phy, value, value & PHY_LANE_CFG_REFCLK0ACK, 100, + RADIO_PORT_TIMEOUT_US); + if (ret) + return ret; + writel(readl(port->phy) | PHY_LANE_CFG_REFCLK1REQ, port->phy); + ret = readl_poll_timeout(port->phy, value, value & PHY_LANE_CFG_REFCLK1ACK, 100, + RADIO_PORT_TIMEOUT_US); + if (ret) + return ret; + writel(readl(port->phy) & ~RADIO_PHY_RESET, port->phy); + writel(readl(port->phy) | BIT(9), port->phy); + writel(readl(port->phy) | BIT(10), port->phy); + apple_pcie_port_rmw_set(port, PORT_PERST_OFF, s->pcie->hw->port_perst); + ret = readl_poll_timeout(port->base + PORT_STATUS, value, value & PORT_STATUS_READY, 100, + 250000); + if (ret) + return ret; + /* Keep the port clock policy that was qualified while the radio was live. */ + value = apple_pcie_port_readl(port, PORT_APPCLK); + apple_pcie_port_writel(port, (value & ~PORT_APPCLK_CGDIS) | (s->appclk & PORT_APPCLK_CGDIS), + PORT_APPCLK); + ret = readl_poll_timeout(port->base + PORT_LINKSTS, value, !(value & PORT_LINKSTS_BUSY), + 100, 250000); + if (ret) + return ret; + writel(readl(dbi + RADIO_DBI_RO_WR) | BIT(0), dbi + RADIO_DBI_RO_WR); + for (unsigned int i = 1; i < ARRAY_SIZE(s->tables); i++) + apple_tunable_apply(dbi, s->tables[i]); + if (speed > 0 && speed <= 4) { + value = readl(dbi + 0x7c); + writel((value & ~PCI_EXP_LNKCAP_SLS) | speed, dbi + 0x7c); + value = readl(dbi + 0x9c); + writel((value & ~GENMASK(6, 1)) | FIELD_PREP(GENMASK(6, 1), (1 << speed) - 1), + dbi + 0x9c); + writew((readw(dbi + 0xa0) & ~PCI_EXP_LNKCTL2_TLS) | speed, dbi + 0xa0); + writel(readl(dbi + RADIO_DBI_WIDTH_SPEED) | BIT(17), dbi + RADIO_DBI_WIDTH_SPEED); + } + value = readl(dbi + RADIO_DBI_LINK_CONTROL); + writel((value & ~GENMASK(21, 16)) | BIT(16), dbi + RADIO_DBI_LINK_CONTROL); + value = readl(dbi + RADIO_DBI_WIDTH_SPEED); + writel((value & ~GENMASK(12, 8)) | BIT(8), dbi + RADIO_DBI_WIDTH_SPEED); + value = readl(dbi + 0x7c); + writel((value & ~PCI_EXP_LNKCAP_MLW) | FIELD_PREP(PCI_EXP_LNKCAP_MLW, 1), dbi + 0x7c); + writel(readl(dbi + RADIO_DBI_RO_WR) & ~BIT(0), dbi + RADIO_DBI_RO_WR); + return 0; +} + +static int apple_radio_routes_restore(struct apple_radio_sleep *s) +{ + struct apple_pcie_port *port = s->port; + const struct hw_info *hw = s->pcie->hw; + + apple_pcie_port_writel(port, s->msicfg & ~PORT_MSICFG_EN, PORT_MSICFG); + for (unsigned int i = 0; i < ARRAY_SIZE(s->rid); i++) { + apple_pcie_port_writel(port, s->rid[i], port_rid2sid_offset(port, i)); + if (apple_pcie_port_readl(port, port_rid2sid_offset(port, i)) != s->rid[i]) + return -EIO; + } + for (unsigned int i = 0; i < ARRAY_SIZE(s->msi); i++) { + apple_pcie_port_writel(port, s->msi[i], hw->port_msimap + 4 * i); + if (apple_pcie_port_readl(port, hw->port_msimap + 4 * i) != s->msi[i]) + return -EIO; + } + apple_pcie_port_writel(port, s->msibase, PORT_MSIBASE); + apple_pcie_port_writel(port, s->msiaddr, hw->port_msiaddr); + apple_pcie_port_writel(port, s->msiaddr_hi, hw->port_msiaddr_hi); + apple_pcie_port_writel(port, s->msicfg, PORT_MSICFG); + if (apple_pcie_port_readl(port, PORT_MSIBASE) != s->msibase || + apple_pcie_port_readl(port, hw->port_msiaddr) != s->msiaddr || + apple_pcie_port_readl(port, hw->port_msiaddr_hi) != s->msiaddr_hi || + apple_pcie_port_readl(port, PORT_MSICFG) != s->msicfg) + return -EIO; + return 0; +} + +static int apple_radio_recover(struct apple_radio_sleep *s) +{ + static const u32 identities[] = { 0x443414e4, 0x5f7214e4 }; + struct apple_pcie_port *port = s->port; + u32 value; + u16 command; + int ret; + + /* These native GPIO operations restore mux and direction as well as level. */ + ret = gpiod_direction_output(s->perst, 1) ?: gpiod_direction_input(s->clkreq); + if (ret) + return ret; + ret = apple_radio_port_restore(s) ?: apple_radio_routes_restore(s); + if (ret) + return ret; + ret = apple_dart_sleep_power_restore(&s->dart); + if (ret) + return ret; + /* Only this transaction's task may access config until the link is checked. */ + s->config_ready = true; + pci_restore_state(s->root); + /* Keep the rail untouched and allow reference clocks to settle. */ + msleep(100); + ret = gpiod_set_value(s->perst, 0); + if (ret) + return ret; + msleep(100); + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); + ret = readl_poll_timeout(port->base + PORT_LINKSTS, value, value & PORT_LINKSTS_UP, 1000, + 500000); + if (ret) + return ret; + msleep(100); + for (unsigned int i = 0; i < ARRAY_SIZE(s->fn); i++) { + if (pci_read_config_dword(s->fn[i].pdev, PCI_VENDOR_ID, &value) || + value != identities[i]) + return -ENODEV; + if (pci_read_config_word(s->fn[i].pdev, PCI_COMMAND, &command) || + (command & PCI_COMMAND_MASTER)) + return -EUCLEAN; + } + apple_pcie_port_writel(port, ~0U, PORT_INTSTAT); + apple_pcie_port_writel(port, s->intmask, PORT_INTMSK); + /* Child noirq callbacks now restore config, including BME, before firmware. */ + return 0; +} + +static int apple_radio_abort(struct apple_radio_sleep *s, u64 generation) +{ + int i, ret = apple_radio_valid(s, generation); + + if (ret) + return ret; + if (irqs_disabled() || s->phase == APPLE_RADIO_VERIFIED || + s->phase == APPLE_RADIO_STOPPING || s->phase == APPLE_RADIO_STOPPED || + s->phase == APPLE_RADIO_RECOVERING) + return -EPERM; + if (s->phase == APPLE_RADIO_ABORTED) + return 0; + if (s->snapshot && apple_radio_snapshot(s, true)) + goto failed; + if (apple_dart_sleep_abort(&s->dart)) + goto failed; + for (i = ARRAY_SIZE(s->fn) - 1; i >= 0; i--) + if (apple_radio_bme_restore(&s->fn[i])) + goto failed; + s->phase = APPLE_RADIO_ABORTED; + /* Publish restored/verified state before admitting host writers. */ + smp_store_release(&s->pcie->radio_closed, false); + return 0; +failed: + s->phase = APPLE_RADIO_FAILED; + return -EUCLEAN; +} + +static int apple_radio_finish(struct apple_radio_sleep *s, u64 generation) +{ + int i, ret = apple_radio_valid(s, generation); + + if (ret) + return ret; + if (s->phase != APPLE_RADIO_VERIFIED && s->phase != APPLE_RADIO_ABORTED) + return -EPERM; + for (i = 0; i < ARRAY_SIZE(s->fn); i++) { + /* Acquire the receipt before reading its published error result. */ + u64 resumed = smp_load_acquire(&s->fn[i].resumed); + + if (READ_ONCE(s->fn[i].error) || + (s->fn[i].suspended == generation && resumed != generation)) + return -EUCLEAN; + } + ret = apple_dart_sleep_finish(&s->dart); + if (ret) + return ret; + /* End ownership before the endpoint finish hooks reopen normal work. */ + smp_store_release(&apple_radio_active, NULL); + for (i = 0; i < ARRAY_SIZE(s->fn); i++) + if (s->fn[i].finish) + s->fn[i].finish(&s->fn[i].pdev->dev); + s->phase = APPLE_RADIO_IDLE; + s->owner = NULL; + return 0; +} + +static int apple_radio_recovery_gpio(struct apple_radio_sleep *s) +{ + static const char *const names[] = { "reset-gpios", "clkreq-gpios" }; + static const unsigned int pins[] = { 163, 160 }; + struct of_phandle_args spec; + struct resource resource; + int ret; + + /* Claim only the two board-defined pins; leave the SMC-owned rail alone. */ + for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { + ret = of_parse_phandle_with_fixed_args(s->port->np, names[i], 2, 0, &spec); + if (ret) + return ret; + ret = of_address_to_resource(spec.np, 0, &resource); + if (!ret && (!of_device_is_compatible(spec.np, "apple,t8132-pinctrl") || + resource.start != 0x39a000000ULL || spec.args[0] != pins[i] || + spec.args[1] != 1)) + ret = -EINVAL; + of_node_put(spec.np); + if (ret) + return ret; + } + s->perst = s->port->perst_gpio; + if (!s->perst) + s->perst = devm_fwnode_gpiod_get(s->pcie->dev, of_fwnode_handle(s->port->np), + "reset", GPIOD_ASIS, "radio PERST#"); + if (IS_ERR(s->perst)) + return PTR_ERR(s->perst); + s->clkreq = devm_fwnode_gpiod_get(s->pcie->dev, of_fwnode_handle(s->port->np), "clkreq", + GPIOD_ASIS, "radio CLKREQ#"); + if (IS_ERR(s->clkreq)) + return PTR_ERR(s->clkreq); + if (gpiod_cansleep(s->perst)) + return -EOPNOTSUPP; + return 0; +} + +/* The retained hierarchy cannot be removed while these references are held. */ +static void apple_radio_put_devices(void *data) +{ + struct apple_radio_sleep *s = data; + + if (READ_ONCE(apple_radio_active) == s) + apple_radio_terminal(); + for (unsigned int i = 0; i < ARRAY_SIZE(s->fn); i++) + pci_dev_put(s->fn[i].pdev); + pci_dev_put(s->root); +} + +static int apple_radio_bind(struct platform_device *pdev, struct apple_radio_sleep *s) +{ + static const u16 ids[] = { 0x4434, 0x5f72 }; + static const char *const names[] = { + "apple,tunable-port", + "apple,tunable-rc", + "apple,tunable-rc-gen3", + "apple,tunable-rc-gen4", + }; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(s->pcie); + struct of_phandle_args map[2] = {}; + struct resource *res; + int ret; + + ret = devm_add_action_or_reset(&pdev->dev, apple_radio_put_devices, s); + if (ret) + return ret; + s->root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); + if (!s->root || !s->root->subordinate) + return -ENODEV; + for (unsigned int i = 0; i < ARRAY_SIZE(s->fn); i++) { + struct pci_dev *ep = pci_get_slot(s->root->subordinate, PCI_DEVFN(0, i)); + + s->fn[i].pdev = ep; + if (!ep || ep->vendor != 0x14e4 || ep->device != ids[i]) + return -ENODEV; + } + ret = apple_radio_recovery_gpio(s); + if (ret) + return ret; + for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, i ? "config" : "port0"); + if (!res) + return -ENOENT; + s->tables[i] = devm_apple_tunable_parse(&pdev->dev, s->port->np, names[i], res); + if (IS_ERR(s->tables[i])) + return PTR_ERR(s->tables[i]); + } + ret = apple_dart_sleep_bind(&pdev->dev, &s->fn[0].pdev->dev, &s->dart); + if (ret) + return ret; + for (unsigned int i = 0; i < ARRAY_SIZE(map); i++) { + ret = of_map_iommu_id(pdev->dev.of_node, 0x100 + i, &map[i]); + if (ret) + break; + if (map[i].np != s->dart.provider->of_node || map[i].args_count != 1 || + map[i].args[0] != 1 + i) { + ret = -EINVAL; + break; + } + } + for (unsigned int i = 0; i < ARRAY_SIZE(map); i++) + of_node_put(map[i].np); + if (ret) + return ret; + s->bound = s->recovery_ready = true; + return 0; +} + +static void apple_radio_register(struct platform_device *pdev, struct apple_pcie *pcie) +{ + struct apple_radio_sleep *s; + int ret; + + if (!pcie->managed_sids) + return; + if (!of_machine_is_compatible("apple,j713") || + !of_device_is_compatible(pcie->dev->of_node, "apple,t8132-pcie") || + pcie->managed_sids != (BIT(1) | BIT(2)) || pcie->piodma_supplier || + !list_is_singular(&pcie->ports) || pcie->nvecs != 32 || !pcie->hw->port_msiaddr_hi || + !pcie->hw->port_msimap) + return; + s = devm_kzalloc(pcie->dev, sizeof(*s), GFP_KERNEL); + if (!s) + return; + s->pcie = pcie; + s->port = list_first_entry(&pcie->ports, struct apple_pcie_port, entry); + if (s->port->idx || !s->port->phy || s->port->sid_map_sz != 16) + return; + ret = apple_radio_bind(pdev, s); + if (ret) { + dev_warn(pcie->dev, "native radio sleep unavailable: %d\n", ret); + return; + } + /* All host callbacks must execute in the task that owns the transaction. */ + device_disable_async_suspend(pcie->dev); + pcie->radio = s; + dev_info(pcie->dev, "native radio sleep recovery registered\n"); +} + +static int apple_radio_prepare(struct device *dev) +{ + static const char *const drivers[] = { "brcmfmac", "hci_bcm4377" }; + struct apple_pcie *pcie = apple_pcie_lookup(dev); + struct apple_radio_sleep *s = pcie->radio; + + if (!pcie->managed_sids || !pm_suspend_via_firmware()) + return 0; + if (!s || !s->bound || s->generation == U64_MAX) + return -EBUSY; + for (unsigned int i = 0; i < ARRAY_SIZE(s->fn); i++) { + struct device *ep = &s->fn[i].pdev->dev; + + if (!ep->driver || strcmp(ep->driver->name, drivers[i])) + return -ENODEV; + } + return apple_radio_begin(s, s->generation + 1); +} + +static int apple_radio_suspend_noirq(struct apple_pcie *pcie) +{ + struct apple_radio_sleep *s = pcie->radio; + int ret; + + if (!pm_suspend_via_firmware()) + return 0; + if (!s || READ_ONCE(apple_radio_active) != s || s->phase != APPLE_RADIO_OPEN) + return -EBUSY; + apple_pcie_port_disable_irq(s->port); + s->host_suspended = true; + ret = apple_radio_seal(s, s->generation); + if (!ret && + (!s->root->state_saved || !s->fn[0].pdev->state_saved || !s->fn[1].pdev->state_saved)) + ret = -EUCLEAN; + if (!ret) + ret = apple_dart_sleep_power_off(&s->dart); + if (ret) { + dev_err(pcie->dev, "native radio suspend refused: %d\n", ret); + if (apple_radio_abort(s, s->generation)) + apple_radio_terminal(); + apple_pcie_port_enable_irq(s->port); + return ret; + } + /* The final SoC commit owns PERST and the port clocks. A refusal leaves them on. */ + s->phase = APPLE_RADIO_STOPPED; + device_set_awake_path(pcie->dev); + dev_info(pcie->dev, "native radio DMA fenced, DART off\n"); + return 0; +} + +static int apple_radio_resume_noirq(struct apple_pcie *pcie) +{ + struct apple_radio_sleep *s = pcie->radio; + int ret; + + if (!s || READ_ONCE(apple_radio_active) != s) + return 0; + if (s->phase == APPLE_RADIO_OPEN || s->phase == APPLE_RADIO_ABORTED) { + apple_pcie_port_enable_irq(s->port); + return 0; + } + if (s->phase != APPLE_RADIO_STOPPED || current != s->owner) + apple_radio_terminal(); + s->phase = APPLE_RADIO_RECOVERING; + if (pm_resume_via_firmware()) { + ret = apple_radio_recover(s); + } else { + /* pm_test or a refused commit: the link stayed up, only the DART was stopped. */ + ret = apple_dart_sleep_power_restore(&s->dart); + s->config_ready = !ret; + for (unsigned int i = 0; !ret && i < ARRAY_SIZE(s->fn); i++) + ret = apple_radio_bme_restore(&s->fn[i]); + } + if (ret) { + dev_emerg(pcie->dev, "native radio recovery failed: %d\n", ret); + s->config_ready = false; + /* Config readers must see the closed gate before the failure state. */ + smp_store_release(&s->phase, APPLE_RADIO_FAILED); + apple_radio_terminal(); + } + /* Publish the complete recovery before config access resumes. */ + smp_store_release(&s->phase, APPLE_RADIO_VERIFIED); + /* Host writers must observe that same restored state. */ + smp_store_release(&pcie->radio_closed, false); + apple_pcie_port_enable_irq(s->port); + if (pm_resume_via_firmware()) + apple_sleep_radio_port_up(); + dev_info(pcie->dev, "native radio %s recovery complete\n", + pm_resume_via_firmware() ? "power-loss" : "retained"); + return 0; +} + +static void apple_radio_complete(struct device *dev) +{ + struct apple_radio_sleep *s = apple_pcie_lookup(dev)->radio; + int ret; + + if (!s || READ_ONCE(apple_radio_active) != s) + return; + if (s->phase == APPLE_RADIO_OPEN || s->phase == APPLE_RADIO_SEALING) { + if (apple_radio_abort(s, s->generation)) + apple_radio_terminal(); + } + ret = apple_radio_finish(s, s->generation); + if (ret) { + dev_emerg(dev, "native radio endpoints failed to resume: %d\n", ret); + apple_radio_terminal(); + } + dev_info(dev, "native radio transaction complete\n"); +} diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 109d54eb6f902..9cb6143bd505b 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -40,6 +40,8 @@ #include #include #include +#include +#include #include #include #include @@ -472,6 +474,8 @@ bool apple_pcie_tunnel_needs_cold_init(struct device_node *np) } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_needs_cold_init); +struct apple_radio_sleep; + struct apple_pcie { struct mutex lock; struct device *dev; @@ -501,6 +505,9 @@ struct apple_pcie { bool resume_failed; /* cleared only by teardown and reprobe */ struct reset_control *reset; u16 managed_sids; + struct apple_radio_sleep *radio; + unsigned int radio_writers; + bool radio_closed; bool neo_config_ready; u8 neo_secondary_bus; struct device *piodma_supplier; @@ -557,6 +564,10 @@ struct apple_pcie_port { unsigned long adopted_slots; /* adopt_link: RID2SID slots routed (slot == SID) */ }; +static bool apple_radio_writer_closed(struct apple_pcie *pcie); +static bool apple_radio_config_allowed(struct apple_pcie *pcie); +static void __noreturn apple_radio_terminal(void); + static int apple_pcie_managed_admit(struct apple_pcie_port *port); static struct apple_pcie *apple_pcie_lookup(struct device *dev); static int apple_pcie_radio_check_cold(struct apple_pcie_port *port); @@ -693,14 +704,21 @@ static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, int ret, hwirq; mutex_lock(&pcie->lock); + if (apple_radio_writer_closed(pcie)) { + mutex_unlock(&pcie->lock); + return -EBUSY; + } + pcie->radio_writers++; hwirq = bitmap_find_free_region(pcie->bitmap, pcie->nvecs, order_base_2(nr_irqs)); mutex_unlock(&pcie->lock); - if (hwirq < 0) - return -ENOSPC; + if (hwirq < 0) { + ret = -ENOSPC; + goto done; + } fwspec.param[fwspec.param_count - 2] += hwirq; @@ -710,7 +728,7 @@ static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, bitmap_release_region(pcie->bitmap, hwirq, order_base_2(nr_irqs)); mutex_unlock(&pcie->lock); - return ret; + goto done; } for (i = 0; i < nr_irqs; i++) { @@ -718,7 +736,12 @@ static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, &apple_msi_bottom_chip, pcie); } - return 0; + ret = 0; +done: + mutex_lock(&pcie->lock); + pcie->radio_writers--; + mutex_unlock(&pcie->lock); + return ret; } static void apple_msi_domain_free(struct irq_domain *domain, unsigned int virq, @@ -728,6 +751,8 @@ static void apple_msi_domain_free(struct irq_domain *domain, unsigned int virq, struct apple_pcie *pcie = domain->host_data; mutex_lock(&pcie->lock); + if (apple_radio_writer_closed(pcie)) + apple_radio_terminal(); bitmap_release_region(pcie->bitmap, d->hwirq, order_base_2(nr_irqs)); @@ -3246,6 +3271,10 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d return err; } mutex_lock(&port->pcie->lock); + if (apple_radio_writer_closed(port->pcie)) { + mutex_unlock(&port->pcie->lock); + return -EBUSY; + } if (port->pcie->managed_sids || port->pcie->hw->sid_indexed_rid2sid) { if (sid >= port->sid_map_sz || @@ -3315,6 +3344,8 @@ static void apple_pcie_disable_device(struct pci_host_bridge *bridge, struct pci return; mutex_lock(&port->pcie->lock); + if (apple_radio_writer_closed(port->pcie)) + apple_radio_terminal(); for_each_set_bit(idx, port->sid_map, port->sid_map_sz) { u32 val; @@ -3378,6 +3409,8 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); struct apple_pcie_port *port; + if (!apple_radio_config_allowed(pcie)) + return NULL; if (pcie->hw->adopt_link) { /* Only adopted ports' root ports, and the endpoint behind each. */ list_for_each_entry(port, &pcie->ports, entry) { @@ -4352,6 +4385,8 @@ static void apple_pcie_neo_sleep_probe(struct platform_device *pdev, struct appl dev_warn(dev, "no deep sleep: the loader didn't publish what this host needs\n"); } +#include "pcie-apple-radio.c" + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -4571,6 +4606,7 @@ static int apple_pcie_probe(struct platform_device *pdev) if (pcie->hw->tunneled) pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); + apple_radio_register(pdev, pcie); return 0; } @@ -5310,12 +5346,11 @@ static int apple_pcie_suspend_noirq(struct device *dev) } /* - * The managed radio has no suspend boundary of its own; suspend-to-idle - * leaves it in place like any other internal port (no power domain). + * The managed J713 radio has a native port/DART recovery transaction. + * Suspend-to-idle leaves its link in place (no power domain). */ if (pcie->managed_sids) - return pm_suspend_target_state == PM_SUSPEND_TO_IDLE || - bringup_no_radio_refuse ? 0 : -EBUSY; + return apple_radio_suspend_noirq(pcie); if (pcie->hw->adopt_link) { int ret = apple_pcie_check_adopted_s2idle(pcie); @@ -5696,6 +5731,8 @@ static int apple_pcie_resume_noirq(struct device *dev) struct apple_pcie_port *port; int ret; + if (pcie->managed_sids) + return apple_radio_resume_noirq(pcie); if (pcie->neo_sleep_armed) return apple_pcie_neo_resume_noirq(pcie); if (pcie->hw->adopt_link) { @@ -5782,6 +5819,8 @@ static int apple_pcie_resume_noirq(struct device *dev) } static const struct dev_pm_ops apple_pcie_pm_ops = { + .prepare = apple_radio_prepare, + .complete = apple_radio_complete, .suspend_noirq = apple_pcie_suspend_noirq, .resume_noirq = apple_pcie_resume_noirq, }; diff --git a/include/linux/soc/apple/radio-sleep.h b/include/linux/soc/apple/radio-sleep.h new file mode 100644 index 0000000000000..873b0ef0846d5 --- /dev/null +++ b/include/linux/soc/apple/radio-sleep.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_SOC_APPLE_RADIO_SLEEP_H +#define _LINUX_SOC_APPLE_RADIO_SLEEP_H + +#include +#include +#include + +struct device; + +/* Endpoint hooks are no-ops outside an exact active managed J713 transaction. */ +#if IS_REACHABLE(CONFIG_PCIE_APPLE) +bool apple_radio_sleep_owned(struct device *dev); +int apple_radio_sleep_endpoint_prepare(struct device *dev, void (*finish)(struct device *dev)); +int apple_radio_sleep_endpoint_suspended(struct device *dev); +int apple_radio_sleep_endpoint_resume(struct device *dev); +void apple_radio_sleep_endpoint_resumed(struct device *dev, int result); +#else +static inline bool apple_radio_sleep_owned(struct device *dev) +{ + return false; +} + +static inline int apple_radio_sleep_endpoint_prepare(struct device *dev, + void (*finish)(struct device *dev)) +{ + return 0; +} + +static inline int apple_radio_sleep_endpoint_suspended(struct device *dev) +{ + return 0; +} + +static inline int apple_radio_sleep_endpoint_resume(struct device *dev) +{ + return 0; +} + +static inline void apple_radio_sleep_endpoint_resumed(struct device *dev, int result) +{ +} +#endif + +#endif diff --git a/tools/asahi/radio-pm-tests.c b/tools/asahi/radio-pm-tests.c new file mode 100644 index 0000000000000..4075d26234405 --- /dev/null +++ b/tools/asahi/radio-pm-tests.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Production lifecycle with failed endpoint, DART and parent transitions. */ +#include +#include +#include +#include +#include +#include +#include +typedef uint16_t u16; +typedef uint64_t u64; +#define ARRAY_SIZE(x) ((int)(sizeof(x) / sizeof((x)[0]))) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +/* Single-threaded fixture: model publication without CPU barriers. */ +#define smp_load_acquire(p) (*(p)) +/* Single-threaded fixture: model publication without CPU barriers. */ +#define smp_store_release(p, v) (*(p) = (v)) +#define PCI_COMMAND 4 +#define PCI_COMMAND_MASTER 4 +#define mutex_lock(p) ((void)(p)) +#define mutex_unlock(p) ((void)(p)) +#define dev_err(...) ((void)0) +#define dev_info(...) ((void)0) +#define dev_emerg(...) ((void)0) +struct device { + int unused; +}; +struct task_struct { + int unused; +} task; +#define current (&task) +struct pci_dev { + struct device dev; + u16 command; + int writes; + bool fail_write, state_saved; +}; +struct apple_radio_sleep; +struct apple_pcie { + struct device *dev; + int lock; + unsigned int radio_writers; + bool radio_closed; + struct apple_radio_sleep *radio; +}; +struct apple_dart_sleep { + bool active, sealed, verified; +}; +enum apple_radio_phase { + APPLE_RADIO_IDLE, + APPLE_RADIO_OPEN, + APPLE_RADIO_SEALING, + APPLE_RADIO_SEALED, + APPLE_RADIO_VERIFIED, + APPLE_RADIO_ABORTED, + APPLE_RADIO_FAILED, + APPLE_RADIO_STOPPING, + APPLE_RADIO_STOPPED, + APPLE_RADIO_RECOVERING +}; +struct apple_radio_function { + struct pci_dev *pdev; + u16 command; + bool attempted, held; + u64 suspended, resumed; + int error; + void (*finish)(struct device *dev); +}; +struct apple_radio_sleep { + struct apple_pcie *pcie; + struct apple_dart_sleep dart; + struct apple_radio_function fn[2]; + struct task_struct *owner; + u64 generation; + enum apple_radio_phase phase; + bool bound, host_suspended, snapshot, recovery_ready, config_ready; + struct pci_dev *root; + void *port; +}; +static struct apple_radio_sleep *apple_radio_active; +static bool irqoff, fw_suspend, fw_resume; +static int seal_error, abort_error, snapshot_error, power_error, recover_error; +static int snapshot_reads, finish_calls, config_reads, off_calls, restore_calls, + recover_calls, port_up; +static bool irq_disabled; +static jmp_buf terminal; +static bool irqs_disabled(void) +{ + return irqoff; +} +static bool pm_suspend_via_firmware(void) +{ + return fw_suspend; +} +static bool pm_resume_via_firmware(void) +{ + return fw_resume; +} +static _Noreturn void apple_radio_terminal(void) +{ + longjmp(terminal, 1); +} +static int pci_read_config_word(struct pci_dev *d, int r, u16 *v) +{ + assert(r == 4); + config_reads++; + *v = d->command; + return 0; +} +static int pci_write_config_word(struct pci_dev *d, int r, u16 v) +{ + assert(r == 4); + d->command = v; + d->writes++; + if (d->fail_write) { + d->fail_write = false; + return 1; + } + return 0; +} +static int pci_wait_for_pending_transaction(struct pci_dev *d) +{ + (void)d; + return 1; +} +static int apple_radio_snapshot(struct apple_radio_sleep *s, bool verify) +{ + (void)s; + (void)verify; + snapshot_reads++; + return snapshot_error; +} +static int apple_dart_sleep_seal(struct apple_dart_sleep *d) +{ + if (seal_error) + return seal_error; + d->sealed = true; + return 0; +} +static int apple_dart_sleep_power_off(struct apple_dart_sleep *d) +{ + assert(d->sealed); + off_calls++; + return power_error; +} +static int apple_dart_sleep_power_restore(struct apple_dart_sleep *d) +{ + restore_calls++; + if (recover_error) + return recover_error; + d->verified = true; + return 0; +} +static int apple_radio_recover(struct apple_radio_sleep *s) +{ + assert(s->phase == APPLE_RADIO_RECOVERING && s->pcie->radio_closed && + irq_disabled); + recover_calls++; + if (recover_error) + return recover_error; + s->dart.verified = true; + s->config_ready = true; + return 0; +} +static int apple_dart_sleep_abort(struct apple_dart_sleep *d) +{ + if (abort_error) + return abort_error; + d->sealed = false; + return 0; +} +static int apple_dart_sleep_finish(struct apple_dart_sleep *d) +{ + d->active = false; + return 0; +} +static void finish_endpoint(struct device *d) +{ + (void)d; + assert(!apple_radio_active); + finish_calls++; +} +static void apple_pcie_port_enable_irq(void *p) +{ + (void)p; + irq_disabled = false; +} +static void apple_pcie_port_disable_irq(void *p) +{ + (void)p; + irq_disabled = true; +} +static void device_set_awake_path(struct device *d) +{ + (void)d; +} +static void apple_sleep_radio_port_up(void) +{ + port_up++; +} +/* ACTUAL_FUNCTIONS */ +static struct apple_radio_sleep s; +static struct apple_pcie host; +static struct pci_dev wifi, bt, root; +static void fresh(void) +{ + memset(&s, 0, sizeof(s)); + memset(&host, 0, sizeof(host)); + memset(&wifi, 0, sizeof(wifi)); + memset(&bt, 0, sizeof(bt)); + wifi.command = bt.command = 7; + s.fn[0].pdev = &wifi; + s.fn[1].pdev = &bt; + s.pcie = &host; + host.radio = &s; + s.bound = true; + s.generation = 9; + s.owner = current; + s.phase = APPLE_RADIO_OPEN; + s.recovery_ready = true; + s.root = &root; + root.state_saved = wifi.state_saved = bt.state_saved = true; + s.host_suspended = true; + s.fn[0].suspended = s.fn[1].suspended = 9; + s.dart.active = true; + apple_radio_active = &s; + irqoff = false; + fw_suspend = true; + fw_resume = false; + irq_disabled = false; + seal_error = abort_error = snapshot_error = power_error = + recover_error = 0; + snapshot_reads = finish_calls = config_reads = off_calls = + restore_calls = recover_calls = port_up = 0; +} +int main(void) +{ + fresh(); + assert(apple_radio_seal(&s, 8) == -ESTALE); + assert(!wifi.writes); + fresh(); + s.fn[1].suspended = 8; + assert(apple_radio_suspend_noirq(&host) == -EBUSY); + assert(!wifi.writes && !host.radio_closed && !off_calls && + !irq_disabled); + fresh(); + host.radio_writers = 1; + assert(apple_radio_suspend_noirq(&host) == -EBUSY); + assert(!wifi.writes); + fresh(); + bt.fail_write = true; + assert(apple_radio_suspend_noirq(&host) == -EIO); + assert(wifi.command == 7 && bt.command == 7 && !host.radio_closed); + fresh(); + seal_error = -EBUSY; + assert(apple_radio_suspend_noirq(&host) == -EBUSY); + assert(wifi.command == 7 && !off_calls); + fresh(); + root.state_saved = false; + assert(apple_radio_suspend_noirq(&host) == -EUCLEAN); + assert(!off_calls && wifi.command == 7); + fresh(); + assert(!apple_radio_suspend_noirq(&host)); + assert(s.phase == APPLE_RADIO_STOPPED && host.radio_closed && + off_calls == 1 && irq_disabled); + assert(!apple_radio_config_allowed(&host)); + assert(wifi.command == 3 && bt.command == 3); + /* Refusal/pm_test: no port reset or retrain, but translations and BME must return. */ + assert(!apple_radio_resume_noirq(&host)); + assert(restore_calls == 1 && !recover_calls && !port_up); + assert(wifi.command == 7 && bt.command == 7 && + s.phase == APPLE_RADIO_VERIFIED && !host.radio_closed && + !irq_disabled); + assert(apple_radio_config_allowed(&host)); + s.fn[0].resumed = s.fn[1].resumed = 9; + s.fn[0].finish = s.fn[1].finish = finish_endpoint; + assert(!apple_radio_finish(&s, 9)); + assert(!apple_radio_active && finish_calls == 2); + /* Real loss: only the full port/routes/DART/link reconstruction admits child config. */ + fresh(); + assert(!apple_radio_suspend_noirq(&host)); + fw_resume = true; + assert(!apple_radio_resume_noirq(&host)); + assert(recover_calls == 1 && !restore_calls && port_up == 1); + assert(apple_radio_config_allowed(&host) && !host.radio_closed && + !irq_disabled); + assert(apple_radio_finish(&s, 9) == + -EUCLEAN); /* Missing endpoint success cannot count. */ + s.fn[0].resumed = s.fn[1].resumed = 9; + s.fn[1].error = -EIO; + assert(apple_radio_finish(&s, 9) == -EUCLEAN); + s.fn[1].error = 0; + assert(!apple_radio_finish(&s, 9)); + for (int physical = 0; physical < 2; physical++) { + fresh(); + assert(!apple_radio_suspend_noirq(&host)); + fw_resume = physical; + recover_error = -EIO; + if (!setjmp(terminal)) { + apple_radio_resume_noirq(&host); + assert(!"failed restore released endpoints"); + } + assert(host.radio_closed && irq_disabled && + s.phase == APPLE_RADIO_FAILED && + !apple_radio_config_allowed(&host)); + } + fresh(); + fw_suspend = false; + assert(!apple_radio_suspend_noirq(&host)); + assert(!off_calls && !wifi.writes); + puts("PASS: radio PM, suspend refusal, DMA fencing and failed recovery"); +} diff --git a/tools/asahi/radio-pm-tests.py b/tools/asahi/radio-pm-tests.py new file mode 100644 index 0000000000000..daf5e44596cc5 --- /dev/null +++ b/tools/asahi/radio-pm-tests.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Execute the native host's real noirq lifecycle, including a refused sleep.""" + +from pathlib import Path +import re, subprocess, tempfile + +root = Path(__file__).resolve().parents[2] +s = (root / "drivers/pci/controller/pcie-apple-radio.c").read_text() +names = ( + "apple_radio_config_allowed", + "apple_radio_bme_hold", + "apple_radio_bme_restore", + "apple_radio_valid", + "apple_radio_commands_held", + "apple_radio_seal", + "apple_radio_abort", + "apple_radio_finish", + "apple_radio_suspend_noirq", + "apple_radio_resume_noirq", +) +functions = [] +for n in names: + m = re.search( + r"^static (?:int|void|bool) " + n + r"\([^;]+?\{.*?\n\}", s, re.M | re.S + ) + assert m, n + functions.append(m.group()) +code = ( + Path(__file__) + .with_suffix(".c") + .read_text() + .replace("/* ACTUAL_FUNCTIONS */", "\n".join(functions)) +) +with tempfile.TemporaryDirectory() as tmp: + p = Path(tmp) / "test.c" + exe = Path(tmp) / "test" + p.write_text(code) + subprocess.run( + ["cc", "-std=gnu11", "-Wall", "-Wextra", "-Werror", str(p), "-o", str(exe)], + check=True, + ) + subprocess.run([str(exe)], check=True) diff --git a/tools/asahi/radio-sleep-fence-tests.c b/tools/asahi/radio-sleep-fence-tests.c new file mode 100644 index 0000000000000..9b8d628d94f6c --- /dev/null +++ b/tools/asahi/radio-sleep-fence-tests.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Failure-first host fixture for the production checked PCI command fence. */ +#include +#include +#include +#include +#include +typedef uint16_t u16; +#define PCI_COMMAND 4 +#define PCI_COMMAND_MASTER 4 +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +struct pci_dev { + u16 command; + int read_count, write_count, fail_read, fail_write; + bool wrong_write; +}; +struct apple_radio_function { + struct pci_dev *pdev; + u16 command; + bool attempted, held; +}; +static int pci_read_config_word(struct pci_dev *d, int reg, u16 *out) +{ + assert(reg == PCI_COMMAND); + if (++d->read_count == d->fail_read) + return 1; + *out = d->command; + return 0; +} +static int pci_write_config_word(struct pci_dev *d, int reg, u16 value) +{ + assert(reg == PCI_COMMAND); + d->write_count++; + d->command = d->wrong_write ? (value ^ 2) : value; + return d->write_count == d->fail_write; +} +/* ACTUAL_FUNCTIONS */ +int main(void) +{ + for (int fault = 0; fault < 6; fault++) { + struct pci_dev d = { .command = 7 }; + struct apple_radio_function f = { .pdev = &d }; + + if (fault == 1) + d.fail_read = 1; + if (fault == 2) + d.command = 0xffff; + if (fault == 3) + d.fail_write = 1; + if (fault == 4) + d.fail_read = 2; + if (fault == 5) + d.wrong_write = true; + int r = apple_radio_bme_hold(&f); + + assert((r == 0) == (fault == 0)); + if (fault == 1 || fault == 2) { + assert(!f.attempted); + assert(d.write_count == 0); + } else + assert(f.attempted); + if (!fault) { + assert(f.held && d.command == 3); + assert(!apple_radio_bme_restore(&f)); + assert(d.command == 7 && !f.held); + } + } + { + struct pci_dev d = { .command = 3 }; + struct apple_radio_function f = { .pdev = &d }; + + assert(!apple_radio_bme_hold(&f)); + assert(!apple_radio_bme_restore(&f)); + assert(d.command == 3); + } + { + struct pci_dev d = { .command = 7 }; + struct apple_radio_function f = { .pdev = &d }; + + assert(!apple_radio_bme_hold(&f)); + d.command = 1; + int w = d.write_count; + + assert(apple_radio_bme_restore(&f) == -EUCLEAN); + assert(d.write_count == w && f.held); + } + { + struct pci_dev d = { .command = 7, .fail_write = 1 }; + struct apple_radio_function f = { .pdev = &d }; + + assert(apple_radio_bme_hold(&f)); + assert(f.attempted); + assert(!apple_radio_bme_restore(&f)); + assert(d.command == 7); + } + puts("PASS: PCI bus-master fence and uncertain-write recovery"); +} diff --git a/tools/asahi/radio-sleep-fence-tests.py b/tools/asahi/radio-sleep-fence-tests.py new file mode 100644 index 0000000000000..b136688d749b4 --- /dev/null +++ b/tools/asahi/radio-sleep-fence-tests.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +from pathlib import Path +import subprocess, tempfile, re + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/pci/controller/pcie-apple-radio.c").read_text() + + +def function(name): + m = re.search(r"^static int " + name + r"\([^;]+?\{.*?\n\}", source, re.M | re.S) + assert m, name + return m.group() + + +actual = "\n".join( + function(n) for n in ("apple_radio_bme_hold", "apple_radio_bme_restore") +) +fixture = ( + Path(__file__) + .with_suffix(".c") + .read_text() + .replace("/* ACTUAL_FUNCTIONS */", actual) +) +with tempfile.TemporaryDirectory() as t: + p = Path(t) / "test.c" + p.write_text(fixture) + b = Path(t) / "test" + subprocess.run( + ["cc", "-std=gnu11", "-Wall", "-Wextra", "-Werror", str(p), "-o", str(b)], + check=True, + ) + subprocess.run([str(b)], check=True) diff --git a/tools/asahi/radio-sleep-power-tests.c b/tools/asahi/radio-sleep-power-tests.c new file mode 100644 index 0000000000000..23124403ed2c4 --- /dev/null +++ b/tools/asahi/radio-sleep-power-tests.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Execute production stop/routing/recovery with a strict DMA/link model. */ +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint64_t u64; +#define __iomem +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(m, v) (((v) << __builtin_ctz(m)) & (m)) +#define BIT(n) (1U << (n)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define READ_ONCE(x) (x) +#define PORT_STATUS 0x804 +#define PORT_STATUS_READY BIT(0) +#define PORT_LINKSTS_BUSY BIT(2) +#define RADIO_DBI_RO_WR 0x8bc +#define RADIO_DBI_LINK_CONTROL 0x710 +#define RADIO_DBI_WIDTH_SPEED 0x80c +#define PCI_EXP_LNKCAP_SLS 0xf +#define PCI_EXP_LNKCTL2_TLS 0xf +#define PCI_EXP_LNKCAP_MLW GENMASK(9, 4) +#define PORT_INTMSK 0x104 +#define PORT_INTSTAT 0x100 +#define PORT_APPCLK 0x800 +#define PORT_APPCLK_EN BIT(0) +#define PORT_APPCLK_CGDIS BIT(8) +#define PORT_LINKSTS 0x208 +#define PORT_LINKSTS_UP BIT(0) +#define PORT_LTSSMCTL 0x80 +#define PORT_LTSSMCTL_START BIT(0) +#define PORT_MSICFG 0x124 +#define PORT_PERST_OFF BIT(0) +#define PORT_MSICFG_EN BIT(0) +#define PORT_MSIBASE 0x168 +#define PHY_LANE_CFG_REFCLK0REQ BIT(0) +#define PHY_LANE_CFG_REFCLK1REQ BIT(1) +#define PHY_LANE_CFG_REFCLK0ACK BIT(2) +#define PHY_LANE_CFG_REFCLK1ACK BIT(3) +#define PHY_LANE_CFG_REFCLKEN (BIT(9) | BIT(10)) +#define RADIO_PORT_L2 0x88 +#define RADIO_PORT_L2_ENTER 0x11 +#define RADIO_PORT_L2_BUSY BIT(0) +#define RADIO_PORT_PERST_GUARD 0x13c +#define RADIO_PHY_RESET BIT(4) +#define RADIO_PHY_LINK BIT(31) +#define RADIO_PORT_TIMEOUT_US 50000 +#define PCI_VENDOR_ID 0 +#define PCI_COMMAND 4 +#define PCI_COMMAND_MASTER 4 +struct device { + int unused; +}; +struct gpio_desc { + int asserted; +}; +struct pci_dev { + u32 identity; + u16 command; + bool state_saved; +}; +struct hw_info { + u32 port_perst, port_msimap, port_msiaddr, port_msiaddr_hi; +}; +struct pci_config_window { + void *win; +}; +struct apple_tunable { + unsigned int id; +}; +struct apple_pcie { + const struct hw_info *hw; + struct pci_config_window *cfg; +}; +struct apple_pcie_port { + struct apple_pcie *pcie; + void *base, *phy, *np; +}; +struct apple_dart_sleep { + bool off, restored; +}; +struct apple_radio_function { + struct pci_dev *pdev; +}; +struct apple_radio_sleep { + struct apple_pcie *pcie; + struct apple_pcie_port *port; + struct apple_dart_sleep dart; + struct gpio_desc *perst, *clkreq; + struct pci_dev *root; + struct apple_radio_function fn[2]; + const struct apple_tunable *tables[4]; + u32 appclk; + bool config_ready; + u32 rid[16], msi[32], msicfg, msibase, msiaddr, msiaddr_hi, intmask; +}; +static u32 portregs[0x8000 / 4], phyregs[0x100 / 4], dbiregs[0x1000 / 4]; +static unsigned int poll_fail, poll_count, table_count, table_at[4], + dbi_open_at, dbi_close_at; +static bool hold_ack; +static bool timeout, drop_route, fail_dart_off, fail_dart_restore, fail_gpio; +static unsigned int ops, gpio_asserted, dart_off_at, clocks_off_at, + dart_restore_at, link_at, route_at; +static unsigned int offset(void *p) +{ + return (char *)p - (char *)portregs; +} +static u32 readl(void *p) +{ + return *(u32 *)p; +} +static void writel(u32 value, void *p) +{ + ops++; + if (p == (void *)phyregs) { + if (!hold_ack) + value = (value & ~(BIT(2) | BIT(3))) | + ((value & (BIT(0) | BIT(1))) << 2); + *(u32 *)p = value; + return; + } + if ((uintptr_t)p >= (uintptr_t)dbiregs && + (uintptr_t)p < (uintptr_t)(dbiregs + ARRAY_SIZE(dbiregs))) { + if (p == (void *)(dbiregs + RADIO_DBI_RO_WR / 4)) { + if (value & 1) + dbi_open_at = ops; + else + dbi_close_at = ops; + } + *(u32 *)p = value; + return; + } + unsigned int off = offset(p); + + assert(off < sizeof(portregs)); + if (drop_route && off == 0x3008) + return; + if (off >= 0x3000) + route_at = ops; + if (off == RADIO_PORT_L2 && !timeout) + value &= ~1U; + if (off == 0x82c && (value & 1)) + portregs[PORT_STATUS / 4] = PORT_STATUS_READY; + if (off == PORT_LTSSMCTL && value == 1) { + assert(dart_restore_at); + link_at = ops; + portregs[PORT_LINKSTS / 4] = 1; + } + if (off == PORT_APPCLK && !(value & PORT_APPCLK_EN)) + clocks_off_at = ops; + *(u32 *)p = value; +} +#define readl_poll_timeout_atomic(p, v, cond, delay, t) \ + ({ \ + (void)(delay); \ + (void)(t); \ + v = readl(p); \ + timeout ? -ETIMEDOUT : ((cond) ? 0 : -ETIMEDOUT); \ + }) +#define readl_poll_timeout(p, v, cond, delay, t) \ + ({ \ + (void)(delay); \ + (void)(t); \ + v = readl(p); \ + poll_count++; \ + poll_fail == poll_count ? -ETIMEDOUT : \ + ((cond) ? 0 : -ETIMEDOUT); \ + }) +#define read_poll_timeout(op, v, cond, delay, t, sleep, p, off) \ + ({ \ + (void)(delay); \ + (void)(t); \ + (void)(sleep); \ + v = op(p, off); \ + timeout ? -ETIMEDOUT : ((cond) ? 0 : -ETIMEDOUT); \ + }) +static u32 apple_pcie_port_readl(struct apple_pcie_port *p, u32 off) +{ + return readl(p->base + off); +} +static void apple_pcie_port_writel(struct apple_pcie_port *p, u32 v, u32 off) +{ + writel(v, p->base + off); +} +static void apple_pcie_port_rmw_set(struct apple_pcie_port *p, u32 bits, + u32 off) +{ + apple_pcie_port_writel(p, apple_pcie_port_readl(p, off) | bits, off); +} +static void apple_pcie_port_rmw_clear(struct apple_pcie_port *p, u32 bits, + u32 off) +{ + apple_pcie_port_writel(p, apple_pcie_port_readl(p, off) & ~bits, off); +} +static u32 port_rid2sid_offset(struct apple_pcie_port *p, int i) +{ + (void)p; + return 0x3000 + 4 * i; +} +static int gpiod_set_value(struct gpio_desc *g, int v) +{ + if (fail_gpio) + return -EIO; + g->asserted = v; + ops++; + if (v) { + gpio_asserted = ops; + portregs[PORT_LINKSTS / 4] = 0; + } + return 0; +} +static int gpiod_direction_output(struct gpio_desc *g, int v) +{ + return gpiod_set_value(g, v); +} +static int gpiod_direction_input(struct gpio_desc *g) +{ + (void)g; + return fail_gpio ? -EIO : 0; +} +static int gpiod_get_value(struct gpio_desc *g) +{ + return g->asserted; +} +static int apple_dart_sleep_power_off(struct apple_dart_sleep *d) +{ + assert(gpio_asserted); + if (fail_dart_off) + return -EIO; + d->off = true; + dart_off_at = ++ops; + return 0; +} +static int apple_dart_sleep_power_restore(struct apple_dart_sleep *d) +{ + assert(route_at); + if (fail_dart_restore) + return -EIO; + d->restored = true; + dart_restore_at = ++ops; + return 0; +} +static int of_pci_get_max_link_speed(void *np) +{ + (void)np; + return 3; +} +static void apple_tunable_apply(void *base, const struct apple_tunable *t) +{ + assert(t->id == table_count++); + table_at[t->id] = ++ops; + if (t->id) { + assert(base == dbiregs && dbi_open_at && !dbi_close_at); + assert(dbiregs[RADIO_DBI_RO_WR / 4] & 1); + } else + assert(base == portregs); +} +static u16 readw(void *p) +{ + return *(u16 *)p; +} +static void writew(u16 v, void *p) +{ + *(u16 *)p = v; + ops++; +} +static void pci_restore_state(struct pci_dev *p) +{ + p->state_saved = false; +} +static int pci_read_config_dword(struct pci_dev *p, int off, u32 *v) +{ + assert(off == PCI_VENDOR_ID); + *v = p->identity; + return 0; +} +static int pci_read_config_word(struct pci_dev *p, int off, u16 *v) +{ + assert(off == PCI_COMMAND); + *v = p->command; + return 0; +} +static void msleep(unsigned int v) +{ + (void)v; +} +#define CONFIG_ARM64 1 +#define USEC_PER_SEC 1000000 +static u64 tick; +static bool stopped_counter, backward_counter; +static u64 arch_timer_read_counter(void) +{ + if (backward_counter) + return tick--; + return stopped_counter ? tick : (tick += 1000); +} +static u32 arch_timer_get_rate(void) +{ + return 1000000; +} +static void cpu_relax(void) +{ +} +/* ACTUAL_SPIN */ +/* ACTUAL_FUNCTIONS */ +static struct hw_info hw = { .port_perst = 0x82c, + .port_msimap = 0x3800, + .port_msiaddr = 0x168, + .port_msiaddr_hi = 0x16c }; +static struct pci_config_window cfg = { .win = dbiregs }; +static struct apple_tunable tables[4] = { { 0 }, { 1 }, { 2 }, { 3 } }; +static struct apple_pcie pcie = { .hw = &hw, .cfg = &cfg }; +static struct apple_pcie_port port = { .pcie = &pcie, + .base = portregs, + .phy = phyregs }; +static struct gpio_desc reset, clkreq; +static struct pci_dev root, wifi, bt; +static struct apple_radio_sleep s; +static void fresh(void) +{ + memset(portregs, 0, sizeof(portregs)); + memset(phyregs, 0, sizeof(phyregs)); + memset(&s, 0, sizeof(s)); + memset(dbiregs, 0, sizeof(dbiregs)); + poll_fail = poll_count = table_count = dbi_open_at = dbi_close_at = 0; + memset(table_at, 0, sizeof(table_at)); + hold_ack = stopped_counter = backward_counter = false; + tick = 0; + for (unsigned int i = 0; i < 4; i++) + s.tables[i] = &tables[i]; + s.appclk = PORT_APPCLK_CGDIS; + s.pcie = &pcie; + s.port = &port; + s.perst = &reset; + s.clkreq = &clkreq; + s.root = &root; + s.fn[0].pdev = &wifi; + s.fn[1].pdev = &bt; + wifi.identity = 0x443414e4; + bt.identity = 0x5f7214e4; + wifi.command = bt.command = 0; + root.state_saved = true; + s.msicfg = 1; + s.msibase = s.msiaddr = 0xfffff000; + s.intmask = 0x1234; + for (unsigned int i = 0; i < 16; i++) + s.rid[i] = 0x80000100 + i; + for (unsigned int i = 0; i < 32; i++) + s.msi[i] = 0x80000000 + i; + portregs[PORT_APPCLK / 4] = PORT_APPCLK_EN; + portregs[PORT_LINKSTS / 4] = 1; + phyregs[0] = ~0U; + timeout = drop_route = fail_dart_off = fail_dart_restore = fail_gpio = + false; + ops = gpio_asserted = dart_off_at = clocks_off_at = dart_restore_at = + link_at = route_at = 0; +} +int main(void) +{ + fresh(); + assert(!apple_radio_recover(&s)); + assert(dart_restore_at < link_at && !reset.asserted); + assert(portregs[PORT_INTMSK / 4] == s.intmask && wifi.command == 0 && + bt.command == 0); + for (unsigned int i = 0; i < 16; i++) + assert(portregs[(0x3000 + 4 * i) / 4] == s.rid[i]); + fresh(); + drop_route = true; + assert(apple_radio_recover(&s) == -EIO); + assert(!dart_restore_at && !link_at && reset.asserted); + fresh(); + fail_dart_restore = true; + assert(apple_radio_recover(&s) == -EIO); + assert(!link_at && reset.asserted); + fresh(); + fail_gpio = true; + assert(apple_radio_recover(&s) == -EIO); + assert(!dart_restore_at && !link_at); + for (unsigned int i = 1; i <= 4; i++) { + fresh(); + poll_fail = i; + assert(apple_radio_recover(&s) == -ETIMEDOUT); + assert(!dart_restore_at && !link_at && reset.asserted && + !dbi_open_at); + } + fresh(); + assert(!apple_radio_recover(&s)); + assert(table_count == 4 && table_at[0] < dbi_open_at && + table_at[3] < dbi_close_at && dbi_close_at < route_at); + assert((dbiregs[0x7c / 4] & PCI_EXP_LNKCAP_SLS) == 3); + assert((dbiregs[0x7c / 4] & PCI_EXP_LNKCAP_MLW) == BIT(4)); + assert((dbiregs[0x9c / 4] & GENMASK(6, 1)) == 14); + assert((*(u16 *)((char *)dbiregs + 0xa0) & PCI_EXP_LNKCTL2_TLS) == 3); + assert(portregs[PORT_APPCLK / 4] & PORT_APPCLK_CGDIS); + fresh(); + tick = 10; + backward_counter = true; + assert(apple_sleep_spin(NULL, 0, 0, 1000) == -EIO); + fresh(); + stopped_counter = true; + assert(apple_sleep_spin(NULL, 0, 0, 1000) == -ETIMEDOUT); + fresh(); + wifi.identity = 0xffffffff; + assert(apple_radio_recover(&s) == -ENODEV); + fresh(); + wifi.command = PCI_COMMAND_MASTER; + assert(apple_radio_recover(&s) == -EUCLEAN); + puts("PASS: port recovery, routing, DART/link ordering and failures"); +} diff --git a/tools/asahi/radio-sleep-power-tests.py b/tools/asahi/radio-sleep-power-tests.py new file mode 100644 index 0000000000000..5b835d0190496 --- /dev/null +++ b/tools/asahi/radio-sleep-power-tests.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +from pathlib import Path +import re +import resource +import subprocess +import tempfile + +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +root = Path(__file__).resolve().parents[2] +src = (root / "drivers/pci/controller/pcie-apple-radio.c").read_text() + + +def extract(name): + m = re.search(r"^static int " + name + r"\([^;]+?\{.*?\n\}", src, re.M | re.S) + assert m, name + return m.group() + + +actual = "\n".join( + extract(n) + for n in ( + "apple_radio_port_restore", + "apple_radio_routes_restore", + "apple_radio_recover", + ) +) +fixture = Path(__file__).with_suffix(".c").read_text() +spin = (root / "include/linux/soc/apple/sleep-spin.h").read_text() +spin = re.search(r"static inline int apple_sleep_spin.*?\n\}", spin, re.S).group() +fixture = fixture.replace("/* ACTUAL_SPIN */", spin) +with tempfile.TemporaryDirectory(prefix="radio-power-") as tmp: + c, exe = Path(tmp) / "test.c", Path(tmp) / "test" + c.write_text(fixture.replace("/* ACTUAL_FUNCTIONS */", actual)) + subprocess.run( + [ + "cc", + "-std=gnu11", + "-Wall", + "-Wextra", + "-Werror", + "-Wno-unused-function", + str(c), + "-o", + str(exe), + ], + check=True, + ) + subprocess.run([str(exe)], check=True) diff --git a/tools/asahi/radio-sleep-snapshot-tests.c b/tools/asahi/radio-sleep-snapshot-tests.c new file mode 100644 index 0000000000000..6a2ef00a5e119 --- /dev/null +++ b/tools/asahi/radio-sleep-snapshot-tests.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +#define BIT(n) (1U << (n)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +struct device { + int unused; +}; +struct hw_info { + u32 port_msimap, port_msiaddr, port_msiaddr_hi; +}; +struct apple_pcie { + struct device *dev; + struct hw_info *hw; + u16 managed_sids; + unsigned int nvecs; + unsigned long *bitmap; +}; +struct apple_pcie_port { + unsigned long sid_map[1]; + unsigned int sid_map_sz; +}; +struct apple_radio_sleep { + struct apple_pcie *pcie; + struct apple_pcie_port *port; + int fn[2]; + u32 rid[16], msi[32], msicfg, msibase, msiaddr, msiaddr_hi, intmask, + appclk; + unsigned long vectors[1], sids[1]; +}; +static u32 regs[0x4000 / 4]; +static bool irqoff; +static unsigned int logs, reads; +static bool irqs_disabled(void) +{ + return irqoff; +} +static void dev_err(struct device *d, const char *fmt, ...) +{ + (void)d; + (void)fmt; + logs++; +} +static u32 apple_pcie_port_readl(struct apple_pcie_port *p, u32 off) +{ + (void)p; + assert(off < sizeof(regs) && !(off & 3)); + reads++; + return regs[off / 4]; +} +static u32 port_rid2sid_offset(struct apple_pcie_port *p, unsigned int i) +{ + (void)p; + assert(i < 16); + return 0x3000 + 4 * i; +} +static bool bitmap_equal(const unsigned long *a, const unsigned long *b, + unsigned int n) +{ + assert(n <= 32); + return ((*a ^ *b) & ((1UL << n) - 1)) == 0; +} +static void bitmap_copy(unsigned long *a, const unsigned long *b, + unsigned int n) +{ + assert(n <= 32); + *a = *b; +} +/* ACTUAL_PRODUCTION */ +int main(void) +{ + unsigned long vectors = 7; + struct hw_info hw = { .port_msimap = 0x3800, + .port_msiaddr = 0x168, + .port_msiaddr_hi = 0x16c }; + struct apple_pcie host = { + .bitmap = &vectors, .hw = &hw, .managed_sids = 6, .nvecs = 32 + }; + struct apple_pcie_port port = { .sid_map_sz = 16, .sid_map = { 6 } }; + struct apple_radio_sleep s = { .pcie = &host, .port = &port }; + + regs[PORT_STATUS / 4] = PORT_STATUS_READY; + /* An operating radio link may assert both UP and BUSY. */ + regs[PORT_LINKSTS / 4] = 0x9b00063d; + regs[0x3004 / 4] = 0x80010100; + regs[0x3008 / 4] = 0x80020101; + regs[PORT_MSICFG / 4] = PORT_MSICFG_EN; + regs[hw.port_msiaddr / 4] = 0xfffff000; + for (unsigned int i = 0; i < 32; i++) + regs[hw.port_msimap / 4 + i] = 0x80000000 | i; + assert(apple_radio_snapshot(&s, false) == 0); + assert(apple_radio_snapshot(&s, true) == 0); + assert(logs == 0); + unsigned int corrupt[] = { + 0x3004, 0x3008, 0x3010, + 0x3800, 0x387c, PORT_MSICFG, + PORT_MSIBASE, hw.port_msiaddr, hw.port_msiaddr_hi + }; + for (unsigned int i = 0; i < ARRAY_SIZE(corrupt); i++) { + regs[corrupt[i] / 4] ^= 0x10; + irqoff = true; + assert(apple_radio_snapshot(&s, true) == -EUCLEAN); + assert(logs == 0); /* No console use in early verification. */ + regs[corrupt[i] / 4] ^= 0x10; + } + host.bitmap[0] ^= 1; + assert(apple_radio_snapshot(&s, true) == -EUCLEAN); + host.bitmap[0] ^= 1; + port.sid_map[0] ^= 1; + assert(apple_radio_snapshot(&s, true) == -EUCLEAN); + port.sid_map[0] ^= 1; + irqoff = false; + regs[PORT_STATUS / 4] = 0; + reads = 0; + assert(apple_radio_snapshot(&s, false) == -EUCLEAN && reads == 1); + regs[PORT_STATUS / 4] = PORT_STATUS_READY; + regs[PORT_LINKSTS / 4] &= ~PORT_LINKSTS_UP; + reads = 0; + assert(apple_radio_snapshot(&s, false) == -EUCLEAN && reads == 2); + regs[PORT_LINKSTS / 4] |= PORT_LINKSTS_UP; + regs[0x3004 / 4] ^= 1; + assert(apple_radio_snapshot(&s, false) == -EUCLEAN); + regs[0x3004 / 4] ^= 1; + regs[PORT_MSICFG / 4] = 0; + assert(apple_radio_snapshot(&s, false) == -EUCLEAN); + puts("PASS: live-link status and RID/MSI snapshot validation"); +} diff --git a/tools/asahi/radio-sleep-snapshot-tests.py b/tools/asahi/radio-sleep-snapshot-tests.py new file mode 100644 index 0000000000000..22678c0f34202 --- /dev/null +++ b/tools/asahi/radio-sleep-snapshot-tests.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise the production retained-link snapshot, including an operating link with BUSY set.""" + +from pathlib import Path +import sleep_test_utils as x +import re +import resource +import subprocess +import tempfile + +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +root = Path(__file__).resolve().parents[2] +radio = (root / "drivers/pci/controller/pcie-apple-radio.c").read_text() +pcie = (root / "drivers/pci/controller/pcie-apple.c").read_text() +constants = ( + "PORT_INTMSK", + "PORT_APPCLK", + "PORT_STATUS", + "PORT_STATUS_READY", + "PORT_LINKSTS", + "PORT_LINKSTS_UP", + "PORT_LINKSTS_BUSY", + "PORT_MSICFG", + "PORT_MSICFG_EN", + "PORT_MSIBASE", + "PORT_RID2SID_VALID", + "PORT_RID2SID_SID_SHIFT", +) +defines = "\n".join( + re.search(r"^#define\s+" + name + r"\s+[^\n]+", pcie, re.M).group() + for name in constants +) +actual = "\n".join( + ( + x.function(pcie, "apple_pcie_managed_word"), + x.function(radio, "apple_radio_snapshot_error"), + x.function(radio, "apple_radio_snapshot"), + ) +) +fixture, main = ( + Path(__file__).with_suffix(".c").read_text().split("/* ACTUAL_PRODUCTION */") +) +with tempfile.TemporaryDirectory(prefix="radio-snapshot-") as tmp: + src, binary = Path(tmp) / "test.c", Path(tmp) / "test" + for negative in (False, True): + code = actual + if negative: + old = "if (!(value & PORT_LINKSTS_UP))" + assert old in code + code = code.replace( + old, + "if ((value & (PORT_LINKSTS_UP | " + "PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP)", + ) + src.write_text(fixture + defines + "\n" + code + main) + subprocess.run( + [ + "cc", + "-std=gnu11", + "-Wall", + "-Wextra", + "-Werror", + str(src), + "-o", + str(binary), + ], + check=True, + ) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if negative: + assert result.returncode != 0, "Old stopped-port check was accepted" + print("PASS: old link-state check rejected by captured-status regression") + else: + assert result.returncode == 0, result.stderr + print(result.stdout.strip())