From 3ac93e79d0699e53a5c83cff96d4ab454b97bce2 Mon Sep 17 00:00:00 2001 From: Erik Bourget Date: Mon, 9 Nov 2026 14:45:02 -0500 Subject: [PATCH] iommu: apple-dart: preserve native radio translations across system sleep The J713 radio DART loses its register state while its live page tables remain in memory. Let the PCIe host own its stop and restore transaction, with mapping writers fenced while bus mastering and the port are held. Restore translation, fault-mask and stream state before admitting DMA. Restrict the provider to the managed J713 radio DART. Reject corrupt snapshots and retain the closed DMA boundary after an incomplete restore. Add fault-injection coverage for writer exclusion, snapshots and power loss. Signed-off-by: Erik Bourget --- MAINTAINERS | 4 + drivers/iommu/apple-dart-power.h | 86 ++++ drivers/iommu/apple-dart-sleep.c | 585 +++++++++++++++++++++++ drivers/iommu/apple-dart.c | 128 ++++- include/linux/soc/apple/dart-sleep.h | 78 +++ include/linux/soc/apple/sleep-spin.h | 41 ++ tools/asahi/dart-sleep-power-tests.c | 350 ++++++++++++++ tools/asahi/dart-sleep-power-tests.py | 61 +++ tools/asahi/dart-sleep-snapshot-tests.c | 153 ++++++ tools/asahi/dart-sleep-snapshot-tests.py | 78 +++ tools/asahi/dart-sleep-writer-tests.c | 52 ++ tools/asahi/dart-sleep-writer-tests.py | 33 ++ tools/asahi/sleep_test_utils.py | 31 ++ 13 files changed, 1674 insertions(+), 6 deletions(-) create mode 100644 drivers/iommu/apple-dart-power.h create mode 100644 drivers/iommu/apple-dart-sleep.c create mode 100644 include/linux/soc/apple/dart-sleep.h create mode 100644 include/linux/soc/apple/sleep-spin.h create mode 100644 tools/asahi/dart-sleep-power-tests.c create mode 100644 tools/asahi/dart-sleep-power-tests.py create mode 100644 tools/asahi/dart-sleep-snapshot-tests.c create mode 100644 tools/asahi/dart-sleep-snapshot-tests.py create mode 100644 tools/asahi/dart-sleep-writer-tests.c create mode 100644 tools/asahi/dart-sleep-writer-tests.py create mode 100644 tools/asahi/sleep_test_utils.py diff --git a/MAINTAINERS b/MAINTAINERS index 0cdac3657b043..7d1c382eff471 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2778,6 +2778,8 @@ F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c F: drivers/input/misc/macsmc-input.c F: drivers/input/touchscreen/apple_z2.c +F: drivers/iommu/apple-dart-power.h +F: drivers/iommu/apple-dart-sleep.c F: drivers/iommu/apple-dart-test.c F: drivers/iommu/apple-dart.c F: drivers/iommu/io-pgtable-dart.c @@ -2824,6 +2826,8 @@ F: include/uapi/drm/asahi_drm.h F: include/uapi/linux/apple-sep.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h +F: tools/asahi/dart-sleep-*-tests.* +F: tools/asahi/sleep_test_utils.py F: tools/testing/selftests/drivers/apple/ ARM/APPLE AVE VIDEO ENCODER DRIVER diff --git a/drivers/iommu/apple-dart-power.h b/drivers/iommu/apple-dart-power.h new file mode 100644 index 0000000000000..b1c9a01720abd --- /dev/null +++ b/drivers/iommu/apple-dart-power.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* T8132 native radio DART power state. The caller owns power and excludes DMA. */ +#ifndef __APPLE_DART_POWER_STATE_H +#define __APPLE_DART_POWER_STATE_H + +#define DART_POWER_TUNABLES 6 +#define DART_POWER_TZ_WORDS 11 +#define DART_POWER_ERROR_PENDING 0x4000 +#define DART_POWER_ERROR_MASK 0x4060 + +struct dart_power_state { + u32 lock, diag, pair[2], version; + u32 tunable[DART_POWER_TUNABLES]; + u32 tz[DART_POWER_TZ_WORDS]; + bool has_tz, valid; +}; + +static const struct { + u32 offset, mask; +} dart_power_tunables[DART_POWER_TUNABLES] = { + { 0x20c, 0x80000121 }, { 0x220, 0x000f0f0f }, { 0x224, 0x00ffffff }, + { 0x300, 0x00001f31 }, { 0x308, 0x3ffffffc }, { 0x310, 0x3ffffffc }, +}; + +static const struct { + u32 offset, mask; +} dart_power_tz[DART_POWER_TZ_WORDS] = { + { 0x500, 7 }, { 0x504, 1 }, { 0x508, 0x3ffffffc }, { 0x510, 0x3ffffffc }, + { 0x518, 0x3ffffffc }, { 0x520, 0x3ffffffc }, { 0x528, 0x3ffffffc }, { 0x530, 0x3ffffffc }, + { 0x538, 0x3ffffffc }, { 0x540, 0x3ffffffc }, { 0x548, 0x3ffffffc }, +}; + +/* Only v2.2/v2.3, with one 32-stream error bitmap, are supported here. */ +static bool dart_power_fault_pending(void __iomem *regs, const struct dart_power_state *s) +{ + return readl(regs + DART_POWER_ERROR_PENDING) || + (s->version >= 0x202 && (readl(regs + 0x704) & 0x777)); +} + +/* Reserved mask bits need not read back as written; complete the posted write. */ +static void dart_power_mask_errors(void __iomem *regs, bool mask) +{ + writel(mask ? ~0U : 0, regs + DART_POWER_ERROR_MASK); + /* Finish the mask store before the ordering read and stream changes. */ + mb(); + readl(regs + DART_POWER_ERROR_MASK); +} + +/* Configuration writes complete and are checked before enabling a stream. */ +static int dart_power_write(void __iomem *regs, u32 offset, u32 value) +{ + writel(value, regs + offset); + return readl(regs + offset) == value ? 0 : -EIO; +} + +static int dart_power_config(void __iomem *regs, u32 offset, u32 mask, u32 saved, bool locked) +{ + u32 value = readl(regs + offset); + + if (!((value ^ saved) & mask)) + return 0; + if (locked) + return -EUCLEAN; + return dart_power_write(regs, offset, (value & ~mask) | (saved & mask)); +} + +static int dart_power_tlb_idle(void __iomem *regs) +{ + u32 value; + int ret; + + ret = readl_poll_timeout_atomic(regs + 0x80, value, !(value & BIT(31)), 1, 10000); + return ret ?: readl_poll_timeout_atomic(regs + 0x604, value, !(value & BIT(0)), 1, 10000); +} + +static int dart_power_flush(void __iomem *regs, bool off) +{ + int ret = dart_power_tlb_idle(regs); + + if (ret) + return ret; + writel(off ? 0xa000 : 0x8000, regs + 0x80); + return dart_power_tlb_idle(regs); +} + +#endif diff --git a/drivers/iommu/apple-dart-sleep.c b/drivers/iommu/apple-dart-sleep.c new file mode 100644 index 0000000000000..dd85d9a166625 --- /dev/null +++ b/drivers/iommu/apple-dart-sleep.c @@ -0,0 +1,585 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by apple-dart.c: J713 radio ownership across system sleep. */ +#include +#include "apple-dart-power.h" + +struct apple_dart_sleep_state { + struct apple_dart *dart; + struct apple_dart_sleep *record; + u64 generation; + u32 tcr[DART_MAX_STREAMS]; + u32 ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; + u32 enabled[BITS_TO_U32(DART_MAX_STREAMS)]; + u32 protect, protect_lock, error_mask, lock; + struct regmap *gapf; + struct dart_power_state power; + bool gapf_on; + bool power_off; +}; + +static int apple_dart_sleep_writer_get(struct apple_dart *dart) +{ + unsigned long flags; + int ret = 0; + + if (!dart->managed_sids) + return 0; + spin_lock_irqsave(&dart->managed_lock, flags); + if (dart->sleep_closed) { + if (dart->sleep_record) + WRITE_ONCE(dart->sleep_record->result, -EUCLEAN); + ret = -EBUSY; + } else if (dart->sleep_writers == UINT_MAX) { + ret = -EOVERFLOW; + } else { + dart->sleep_writers++; + } + spin_unlock_irqrestore(&dart->managed_lock, flags); + return ret; +} + +static void apple_dart_sleep_writer_put(struct apple_dart *dart) +{ + unsigned long flags; + + if (!dart->managed_sids) + return; + spin_lock_irqsave(&dart->managed_lock, flags); + dart->sleep_writers--; + spin_unlock_irqrestore(&dart->managed_lock, flags); +} + +/* Close only at zero users, so an admitted writer may nest a TLB operation. */ +static int apple_dart_sleep_close(struct apple_dart *dart) +{ + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&dart->managed_lock, flags); + if (dart->sleep_closed || dart->sleep_writers) + ret = -EBUSY; + else + dart->sleep_closed = true; + spin_unlock_irqrestore(&dart->managed_lock, flags); + return ret; +} + +/* A void unmap/free caller may release backing memory when we return. */ +static void __noreturn apple_dart_sleep_release_failed(void) +{ + panic("J713 radio: DMA state cannot be released safely"); +} + +static int apple_dart_sleep_valid(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + + if (!sleep || !sleep->private || !sleep->active || sleep->owner != current) + return -ESTALE; + s = sleep->private; + if (s->record != sleep || s->dart->sleep_record != sleep || !sleep->generation || + s->generation != sleep->generation) + return -ESTALE; + return READ_ONCE(sleep->result) ?: (READ_ONCE(s->dart->managed_failed) ? -EUCLEAN : 0); +} + +static void apple_dart_sleep_unlink(void *data) +{ + device_link_del(data); +} + +int apple_dart_sleep_bind(struct device *backend, struct device *client, + struct apple_dart_sleep *sleep) +{ + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(client); + struct device_link *link; + struct apple_dart_sleep_state *s; + struct apple_dart *dart; + int ret; + + if (!of_machine_is_compatible("apple,j713") || !cfg || !cfg->stream_maps[0].dart || + cfg->stream_maps[1].dart) + return -ENODEV; + dart = cfg->stream_maps[0].dart; + if ((!dart->managed_sids) || dart->hw->type != DART_T8110 || !READ_ONCE(dart->iommu.ready)) + return -EINVAL; + link = device_link_add(backend, dart->dev, DL_FLAG_STATELESS); + if (!link) + return -EINVAL; + ret = devm_add_action_or_reset(backend, apple_dart_sleep_unlink, link); + if (ret) + return ret; + s = devm_kzalloc(backend, sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->dart = dart; + s->record = sleep; + if (dart->managed_sids) { + struct resource res; + + if (of_address_to_resource(dart->dev->of_node, 0, &res) || + res.start != 0x490000000ULL || resource_size(&res) != 0x20000) + return -EINVAL; + s->gapf = syscon_regmap_lookup_by_compatible("apple,t8132-ps-prot"); + if (IS_ERR(s->gapf)) + return PTR_ERR(s->gapf); + } + *sleep = (struct apple_dart_sleep){ + .private = s, + .provider = dart->dev, + .client = client, + }; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_bind); + +int apple_dart_sleep_begin(struct apple_dart_sleep *sleep, u64 generation) +{ + struct apple_dart_sleep_state *s; + struct apple_dart *dart; + unsigned long flags; + int ret = 0; + + if (!sleep || !sleep->private || !generation || sleep->active || in_interrupt() || + irqs_disabled() || pm_suspend_target_state != PM_SUSPEND_MEM) + return -EINVAL; + s = sleep->private; + dart = s->dart; + if (s->record != sleep) + return -ESTALE; + spin_lock_irqsave(&dart->managed_lock, flags); + if (dart->sleep_record || dart->sleep_closed || dart->managed_failed) { + ret = -EBUSY; + } else { + s->generation = generation; + sleep->generation = generation; + sleep->owner = current; + sleep->result = 0; + sleep->sealed = false; + sleep->verified = false; + sleep->active = true; + s->power.valid = s->power_off = false; + dart->sleep_record = sleep; + } + spin_unlock_irqrestore(&dart->managed_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_begin); + +static int apple_dart_sleep_snapshot_error(struct apple_dart *dart, const char *field, + unsigned int index, u32 value, u32 expected) +{ + /* Leave diagnostics silent if a caller has disabled interrupts. */ + if (!irqs_disabled()) + dev_err(dart->dev, "sleep %s[%u]: observed=%#x expected=%#x\n", field, index, value, + expected); + return -EUCLEAN; +} + +static int apple_dart_sleep_snapshot(struct apple_dart_sleep_state *s, bool verify) +{ + struct apple_dart *dart = s->dart; + unsigned int sid, idx, count = dart->hw->ttbr_count; + u32 value; + + if (READ_ONCE(dart->managed_failed)) + return apple_dart_sleep_snapshot_error(dart, "managed-failed", 0, 1, 0); + if (dart->managed_sids && + (apple_dart_fault_pending(dart) || + (apple_dart_readl(dart, DART_T8110_TLB_CMD) & DART_T8110_TLB_CMD_BUSY))) + return -EBUSY; + value = apple_dart_readl(dart, dart->hw->lock); + if (verify && value != s->lock) + return apple_dart_sleep_snapshot_error(dart, "lock", 0, value, s->lock); + if (!verify) + s->lock = value; + for (sid = 0; sid < dart->num_streams; sid++) { + value = apple_dart_readl(dart, DART_TCR(dart, sid)); + if (verify && value != s->tcr[sid]) + return apple_dart_sleep_snapshot_error(dart, "tcr", sid, value, + s->tcr[sid]); + if (!verify) + s->tcr[sid] = value; + for (idx = 0; idx < count; idx++) { + value = apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); + if (verify && value != s->ttbr[sid][idx]) + return apple_dart_sleep_snapshot_error( + dart, "ttbr", sid * count + idx, value, s->ttbr[sid][idx]); + if (!verify) + s->ttbr[sid][idx] = value; + } + } + if (!dart->managed_sids) + return 0; + for (idx = 0; idx < BITS_TO_U32(dart->num_streams); idx++) { + value = apple_dart_readl(dart, dart->hw->enable_streams + 4 * idx); + if (verify && value != s->enabled[idx]) + return apple_dart_sleep_snapshot_error(dart, "enabled", idx, value, + s->enabled[idx]); + if (!verify) + s->enabled[idx] = value; + } + value = apple_dart_readl(dart, DART_T8110_PROTECT); + if (verify && value != s->protect) + return apple_dart_sleep_snapshot_error(dart, "protect", 0, value, s->protect); + if (!verify) + s->protect = value; + value = apple_dart_readl(dart, DART_T8110_PROTECT_LOCK); + if (verify && value != s->protect_lock) + return apple_dart_sleep_snapshot_error(dart, "protect-lock", 0, value, + s->protect_lock); + if (!verify) + s->protect_lock = value; + value = apple_dart_readl(dart, dart->version >= 0x0202 ? DART_T8110_V2_ERROR_MASK : + DART_T8110_ERROR_MASK); + if (verify && value != s->error_mask) + return apple_dart_sleep_snapshot_error(dart, "error-mask", 0, value, s->error_mask); + if (!verify) + s->error_mask = value; + if (s->error_mask) + return apple_dart_sleep_snapshot_error(dart, "error-mask-admission", 0, + s->error_mask, 0); + if (s->protect & DART_T8110_PROTECT_TTBR_TCR) + return apple_dart_sleep_snapshot_error(dart, "protect-admission", 0, s->protect, 0); + for (sid = 1; sid <= 2; sid++) { + bool live = false; + + /* Admit both supported table depths, but no bypass or remapping. */ + value = s->tcr[sid]; + if (dart->four_level) + value &= ~dart->hw->tcr_4level; + if (value != DART_T8110_TCR_TRANSLATE_ENABLE) + return apple_dart_sleep_snapshot_error(dart, "translation-admission", sid, + s->tcr[sid], + DART_T8110_TCR_TRANSLATE_ENABLE); + for (idx = 0; idx < count; idx++) + live |= !!(s->ttbr[sid][idx] & dart->hw->ttbr_valid); + if (!live) + return apple_dart_sleep_snapshot_error( + dart, "live-root", sid, s->ttbr[sid][0], dart->hw->ttbr_valid); + } + return 0; +} + +/* The native PCIe DART uses slice 28; no other slice or policy bit is changed. */ +static int apple_dart_sleep_gapf(struct apple_dart_sleep_state *s, bool on) +{ + u32 value; + int ret; + + ret = regmap_read(s->gapf, APPLE_PS_PROT_SLICE(28), &value); + if (ret) + return ret; + if (!(value & BIT(1))) + return -EPERM; + ret = regmap_update_bits(s->gapf, APPLE_PS_PROT_SLICE(28), BIT(4), on ? BIT(4) : 0); + if (!ret) + ret = regmap_read(s->gapf, APPLE_PS_PROT_SLICE(28), &value); + return ret ?: (!!(value & BIT(4)) == on ? 0 : -EIO); +} + +static int apple_dart_sleep_power_capture(struct apple_dart_sleep_state *s) +{ + struct apple_dart *dart = s->dart; + struct dart_power_state *p = &s->power; + u32 value; + int ret; + + p->valid = false; + if (!s->gapf || dart->managed_sids != (BIT(1) | BIT(2)) || dart->hw->type != DART_T8110 || + dart->hw->ttbr_count != 1 || dart->num_streams < 19 || dart->num_streams > 32 || + dart->version < 0x202 || dart->version > 0x203) + return -EOPNOTSUPP; + ret = regmap_read(s->gapf, APPLE_PS_PROT_SLICE(28), &value); + if (ret) + return ret; + if (!(value & BIT(1))) + return -EPERM; + /* A working native DART may inherit clock protection disabled. */ + s->gapf_on = !!(value & BIT(4)); + p->version = dart->version; + if (dart_power_fault_pending(dart->regs, p)) + return -EBUSY; + p->lock = s->protect_lock; + p->diag = apple_dart_readl(dart, 0x210); + p->pair[0] = apple_dart_readl(dart, 0x228); + p->pair[1] = apple_dart_readl(dart, 0x22c); + p->has_tz = apple_dart_readl(dart, 0x4) & BIT(2); + if (p->has_tz) { + if ((p->lock & GENMASK(4, 3)) != GENMASK(4, 3)) + return -EPERM; + for (unsigned int i = 0; i < DART_POWER_TZ_WORDS; i++) + p->tz[i] = apple_dart_readl(dart, dart_power_tz[i].offset) & + dart_power_tz[i].mask; + } + for (unsigned int i = 0; i < DART_POWER_TUNABLES; i++) + p->tunable[i] = apple_dart_readl(dart, dart_power_tunables[i].offset); + p->valid = true; + return 0; +} + +int apple_dart_sleep_power_ready(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + int ret = apple_dart_sleep_valid(sleep); + + if (ret) + return ret; + s = sleep->private; + if (!sleep->sealed || sleep->verified || !s->dart->sleep_closed || s->power_off) + return -EPERM; + return s->power.valid ? 0 : -ENODATA; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_power_ready); + +static int apple_dart_sleep_tlb_idle(void __iomem *regs) +{ + return apple_sleep_spin(regs + 0x80, BIT(31), 0, 10000) ?: + apple_sleep_spin(regs + 0x604, BIT(0), 0, 10000); +} + +static int apple_dart_sleep_idle(struct apple_dart_sleep_state *s) +{ + struct apple_dart *dart = s->dart; + u32 params = readl(dart->regs + DART_PARAMS2); + unsigned int ports = max(FIELD_GET(DART_T8110_PARAMS2_READ_PORTS, params), + FIELD_GET(DART_T8110_PARAMS2_WRITE_PORTS, params)); + int ret; + + if (!ports || ports > 4 || dart_power_fault_pending(dart->regs, &s->power)) + return -EIO; + ret = apple_dart_sleep_tlb_idle(dart->regs); + if (ret) + return ret; + for (unsigned int port = 0; port < ports; port++) { + ret = apple_sleep_spin(dart->regs + DART_T8110_OUTSTANDING(port), ~0U, 0, + DART_T8110_IDLE_TIMEOUT_US); + if (ret) + return ret; + } + return dart_power_fault_pending(dart->regs, &s->power) ? -EIO : 0; +} + +/* The owner has stopped its requesters; port clocks still make the DART usable. */ +int apple_dart_sleep_power_off(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + struct apple_dart *dart; + int ret = apple_dart_sleep_power_ready(sleep); + + if (ret) + return ret; + if (in_interrupt() || irqs_disabled()) + return -EPERM; + s = sleep->private; + dart = s->dart; + ret = apple_dart_sleep_snapshot(s, true); + if (ret) + return ret; + ret = apple_dart_sleep_idle(s); + if (ret) + return ret; + if (dart_power_fault_pending(dart->regs, &s->power)) + return -EBUSY; + /* The fault IRQ must not access the window after the owner drops clocks. */ + WRITE_ONCE(dart->commands_gated, true); + dart_power_mask_errors(dart->regs, true); + for (unsigned int sid = 0; sid < dart->num_streams; sid++) { + if (s->enabled[sid / 32] & BIT(sid % 32)) { + apple_dart_writel(dart, BIT(sid % 32), + DART_T8110_DISABLE_STREAMS + 4 * (sid / 32)); + /* Complete each stream-disable command before the next one. */ + mb(); + apple_dart_readl(dart, DART_T8110_DISABLE_STREAMS + 4 * (sid / 32)); + } + } + for (unsigned int i = 0; i < BITS_TO_U32(dart->num_streams); i++) + if (apple_dart_readl(dart, dart->hw->enable_streams + 4 * i)) { + ret = -EIO; + goto fail; + } + ret = apple_dart_sleep_tlb_idle(dart->regs); + if (ret) + goto fail; + writel(0xa000, dart->regs + 0x80); + ret = apple_dart_sleep_tlb_idle(dart->regs) ?: apple_dart_sleep_gapf(s, false); + if (ret) + goto fail; + s->power_off = true; + return 0; +fail: + sleep->result = ret; + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_power_off); + +/* Called after parent and port clocks, while PERST and endpoint BME stay held. */ +int apple_dart_sleep_power_restore(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + struct apple_dart *dart; + struct dart_power_state *p; + u32 protect, lock; + bool locked; + int ret = apple_dart_sleep_valid(sleep); + + if (ret) + return ret; + s = sleep->private; + dart = s->dart; + p = &s->power; + if (in_interrupt() || irqs_disabled() || !sleep->sealed || sleep->verified || + !s->power_off || !p->valid || !READ_ONCE(dart->commands_gated) || + !READ_ONCE(dart->sleep_closed)) + return -EPERM; + ret = apple_dart_sleep_gapf(s, s->gapf_on); + if (ret) + goto fail; + for (unsigned int i = 0; i < BITS_TO_U32(dart->num_streams); i++) + if (apple_dart_readl(dart, dart->hw->enable_streams + 4 * i)) { + ret = -EBUSY; + goto fail; + } + if (dart_power_fault_pending(dart->regs, p)) { + ret = -EBUSY; + goto fail; + } + protect = apple_dart_readl(dart, DART_T8110_PROTECT); + lock = apple_dart_readl(dart, DART_T8110_PROTECT_LOCK); + locked = protect & lock & BIT(0); + if ((lock & GENMASK(4, 3)) != (p->lock & GENMASK(4, 3)) || + !!(apple_dart_readl(dart, 0x4) & BIT(2)) != p->has_tz) { + ret = -EUCLEAN; + goto fail; + } + dart_power_mask_errors(dart->regs, true); + ret = dart_power_write(dart->regs, 0x210, p->diag); + if (ret) + goto fail; + if (p->has_tz) + for (unsigned int i = 0; i < DART_POWER_TZ_WORDS; i++) + if ((apple_dart_readl(dart, dart_power_tz[i].offset) & + dart_power_tz[i].mask) != p->tz[i]) { + ret = -EUCLEAN; + goto fail; + } + for (unsigned int i = 0; i < DART_POWER_TUNABLES; i++) { + bool apf_locked = dart_power_tunables[i].offset >= 0x300 && + (protect & lock & BIT(2)); + + ret = dart_power_config(dart->regs, dart_power_tunables[i].offset, + dart_power_tunables[i].mask, p->tunable[i], + locked || apf_locked); + if (ret) + goto fail; + } + for (unsigned int sid = 0; sid < dart->num_streams; sid++) { + ret = dart_power_config(dart->regs, DART_TCR(dart, sid), ~0U, + s->tcr[sid], locked) ?: + dart_power_config(dart->regs, DART_TTBR(dart, sid, 0), ~0U, + s->ttbr[sid][0], locked); + if (ret) + goto fail; + } + ret = dart_power_flush(dart->regs, false) ?: + dart_power_config(dart->regs, 0x228, ~0U, p->pair[0], locked) ?: + dart_power_config(dart->regs, 0x22c, ~0U, p->pair[1], locked); + if (ret) + goto fail; + dart_power_mask_errors(dart->regs, false); + for (unsigned int i = 0; i < BITS_TO_U32(dart->num_streams); i++) { + ret = dart_power_write(dart->regs, dart->hw->enable_streams + 4 * i, s->enabled[i]); + if (ret) + goto fail; + } + dma_wmb(); + sleep->verified = true; + /* Publish the complete translation context before IRQ or mapping users. */ + smp_store_release(&dart->commands_gated, false); + /* Mapping writers acquire the same completed restore through this gate. */ + smp_store_release(&dart->sleep_closed, false); + return 0; +fail: + sleep->result = ret; + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_power_restore); + +int apple_dart_sleep_seal(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + int ret = apple_dart_sleep_valid(sleep); + + if (ret) + return ret; + if (irqs_disabled() || sleep->sealed) + return -EPERM; + s = sleep->private; + ret = apple_dart_sleep_close(s->dart); + if (ret) + return ret; + /* No table publication/invalidation is still in flight beyond this gate. */ + dma_wmb(); + ret = apple_dart_sleep_snapshot(s, false); + if (!ret && s->dart->managed_sids) + ret = apple_dart_sleep_power_capture(s); + if (ret) { + /* Corrupt live roots must not regain DMA even on pre-entry unwind. */ + if (ret == -EUCLEAN) { + sleep->result = -EUCLEAN; + return ret; + } + /* Publish cancellation before allowing table writers again. */ + smp_store_release(&s->dart->sleep_closed, false); + return ret; + } + sleep->sealed = true; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_seal); + +int apple_dart_sleep_abort(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + int ret = apple_dart_sleep_valid(sleep); + + if (ret) + return ret; + if (irqs_disabled() || sleep->verified) + return -EPERM; + s = sleep->private; + if (s->power_off || READ_ONCE(s->dart->commands_gated)) + return -EPERM; + if (sleep->sealed) { + ret = apple_dart_sleep_snapshot(s, true); + if (ret) { + sleep->result = -EUCLEAN; + return -EUCLEAN; + } + } + /* The saved translation state was checked before reopening writers. */ + smp_store_release(&s->dart->sleep_closed, false); + sleep->sealed = false; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_abort); + +int apple_dart_sleep_finish(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + unsigned long flags; + int ret = apple_dart_sleep_valid(sleep); + + if (ret) + return ret; + s = sleep->private; + if (irqs_disabled() || READ_ONCE(s->dart->sleep_closed)) + return -EPERM; + spin_lock_irqsave(&s->dart->managed_lock, flags); + s->dart->sleep_record = NULL; + sleep->active = false; + sleep->owner = NULL; + spin_unlock_irqrestore(&s->dart->managed_lock, flags); + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_finish); diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 87d2ccef7f2b7..c60c45477b9ec 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -154,6 +155,7 @@ #define DART_T8110_ERROR_FLAG BIT(31) #define DART_T8110_ERROR_MASK 0x104 +#define DART_T8110_V2_ERROR_MASK 0x4060 #define DART_T8110_ERROR_READ_FAULT BIT(5) #define DART_T8110_ERROR_WRITE_FAULT BIT(4) @@ -457,6 +459,9 @@ struct apple_dart { bool managed_failed; /* Serialize owned TCR publication against IRQ quarantine. */ spinlock_t managed_lock; + struct apple_dart_sleep *sleep_record; + unsigned int sleep_writers; + bool sleep_closed; DECLARE_BITMAP(remapped, DART_MAX_STREAMS); u8 remap_target[DART_MAX_STREAMS]; @@ -493,6 +498,10 @@ struct apple_dart { int locked_retire_err[DART_MAX_STREAMS]; }; +static int apple_dart_sleep_writer_get(struct apple_dart *dart); +static void apple_dart_sleep_writer_put(struct apple_dart *dart); +static void __noreturn apple_dart_sleep_release_failed(void); + #ifndef MODULE static struct kexec_blocker apple_dart_radio_kexec = { .users = ATOMIC_INIT(0), @@ -2336,8 +2345,11 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, err = err ?: ret; continue; } + if (apple_dart_sleep_writer_get(stream.dart)) + apple_dart_sleep_release_failed(); ret = pm_runtime_resume_and_get(stream.dart->dev); if (ret < 0) { + apple_dart_sleep_writer_put(stream.dart); dev_err_ratelimited(stream.dart->dev, "failed to power DART for TLB synchronization: %d\n", ret); err = err ?: ret; @@ -2364,6 +2376,7 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, if (ret && stream.dart->managed_sids) apple_dart_managed_quarantine(stream.dart); pm_runtime_put(stream.dart->dev); + apple_dart_sleep_writer_put(stream.dart); err = err ?: ret; } return err; @@ -2566,7 +2579,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, return apple_dart_inherited_phys(dart_domain, address); } -static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, +static int apple_dart_map_pages_unguarded(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) @@ -2600,7 +2613,26 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, mapped); } -static size_t apple_dart_unmap_pages(struct iommu_domain *domain, +static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct apple_dart_domain *cfg = to_dart_domain(domain); + struct apple_dart *dart = cfg->stream_maps[0].dart; + int result; + int gate; + + gate = dart ? apple_dart_sleep_writer_get(dart) : 0; + if (gate) + return gate; + result = apple_dart_map_pages_unguarded(domain, iova, paddr, pgsize, pgcount, + prot, gfp, mapped); + if (dart) + apple_dart_sleep_writer_put(dart); + return result; +} + +static size_t apple_dart_unmap_pages_unguarded(struct iommu_domain *domain, unsigned long iova, size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) @@ -2616,6 +2648,23 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain, gather); } +static size_t apple_dart_unmap_pages(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct apple_dart_domain *cfg = to_dart_domain(domain); + struct apple_dart *dart = cfg->stream_maps[0].dart; + size_t result; + int gate; + + gate = dart ? apple_dart_sleep_writer_get(dart) : 0; + if (gate) + apple_dart_sleep_release_failed(); + result = apple_dart_unmap_pages_unguarded(domain, iova, pgsize, pgcount, gather); + if (dart) + apple_dart_sleep_writer_put(dart); + return result; +} + static int apple_dart_setup_translation(struct apple_dart_domain *domain, struct apple_dart_stream_map *stream_map) @@ -2890,7 +2939,7 @@ static int apple_dart_domain_add_streams(struct apple_dart_domain *domain, true); } -static int apple_dart_attach_dev_paging(struct iommu_domain *domain, +static int apple_dart_attach_dev_paging_unguarded(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) { @@ -2950,7 +2999,24 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain, return ret; } -static int apple_dart_attach_dev_identity(struct iommu_domain *domain, +static int apple_dart_attach_dev_paging(struct iommu_domain *domain, struct device *dev, + struct iommu_domain *old) +{ + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + struct apple_dart *dart = cfg->stream_maps[0].dart; + int result; + int gate; + + gate = dart ? apple_dart_sleep_writer_get(dart) : 0; + if (gate) + return gate; + result = apple_dart_attach_dev_paging_unguarded(domain, dev, old); + if (dart) + apple_dart_sleep_writer_put(dart); + return result; +} + +static int apple_dart_attach_dev_identity_unguarded(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) { @@ -2976,6 +3042,23 @@ static int apple_dart_attach_dev_identity(struct iommu_domain *domain, return 0; } +static int apple_dart_attach_dev_identity(struct iommu_domain *domain, struct device *dev, + struct iommu_domain *old) +{ + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + struct apple_dart *dart = cfg->stream_maps[0].dart; + int result; + int gate; + + gate = dart ? apple_dart_sleep_writer_get(dart) : 0; + if (gate) + return gate; + result = apple_dart_attach_dev_identity_unguarded(domain, dev, old); + if (dart) + apple_dart_sleep_writer_put(dart); + return result; +} + static const struct iommu_domain_ops apple_dart_identity_ops = { .attach_dev = apple_dart_attach_dev_identity, }; @@ -2985,7 +3068,7 @@ static struct iommu_domain apple_dart_identity_domain = { .ops = &apple_dart_identity_ops, }; -static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, +static int apple_dart_attach_dev_blocked_unguarded(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) { @@ -3007,6 +3090,23 @@ static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, return 0; } +static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, struct device *dev, + struct iommu_domain *old) +{ + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + struct apple_dart *dart = cfg->stream_maps[0].dart; + int result; + int gate; + + gate = dart ? apple_dart_sleep_writer_get(dart) : 0; + if (gate) + return gate; + result = apple_dart_attach_dev_blocked_unguarded(domain, dev, old); + if (dart) + apple_dart_sleep_writer_put(dart); + return result; +} + static const struct iommu_domain_ops apple_dart_blocked_ops = { .attach_dev = apple_dart_attach_dev_blocked, }; @@ -3055,6 +3155,8 @@ static void apple_dart_release_device(struct device *dev) struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); for_each_stream_map(j, cfg, stream_map) { + if (READ_ONCE(stream_map->dart->sleep_record)) + apple_dart_sleep_release_failed(); /* Borrowed display roots stay exactly as the DCP left them. */ if (!stream_map->dart->locked || apple_dart_has_borrowed(stream_map->dart)) continue; @@ -3128,7 +3230,7 @@ static void apple_dart_domain_free(struct iommu_domain *domain) return; } for_each_stream_map(i, dart_domain, map) { - if (map->dart->managed_sids) + if (map->dart->managed_sids || READ_ONCE(map->dart->sleep_record)) return; if (!map->dart->locked) continue; @@ -5511,6 +5613,8 @@ static const struct apple_dart_hw apple_dart_hw_t8110 = { .ttbr_count = 1, }; +#include "apple-dart-sleep.c" + static void apple_dart_save_state(struct apple_dart *dart) { unsigned int sid, idx; @@ -5650,6 +5754,10 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); + /* Native radio owner restores before the PCI functions resume. */ + if (dart->managed_sids && READ_ONCE(dart->sleep_record)) + return 0; + dart->tunnel_state_restored = false; /* Its consumer saved this already and restores it: see apple_dart_sleep_prepare(). */ if (dart->sleep_saved) @@ -5801,6 +5909,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) unsigned int sid, idx; int ret; + /* Native radio owner restores before the PCI functions resume. */ + if (dart->managed_sids && READ_ONCE(dart->sleep_record)) + return 0; + /* No reset or restore: see apple_dart_suspend(). */ if (dart->cold_ane || dart->sleep_saved) return 0; @@ -5904,6 +6016,10 @@ static __maybe_unused int apple_dart_resume_noirq(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); int ret; + /* Its host restores every register after the native port clocks return. */ + if (dart->managed_sids && READ_ONCE(dart->sleep_record)) + return pm_runtime_force_resume(dev); + /* * After a real sleep a DART clocked by its host's port reads nothing until the host * sets the port up again. Gate it, so that nothing touches it, until then. diff --git a/include/linux/soc/apple/dart-sleep.h b/include/linux/soc/apple/dart-sleep.h new file mode 100644 index 0000000000000..6e41b324b4dae --- /dev/null +++ b/include/linux/soc/apple/dart-sleep.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_SOC_APPLE_DART_SLEEP_H +#define _LINUX_SOC_APPLE_DART_SLEEP_H + +#include +#include +#include + +struct device; +struct task_struct; + +/* Caller-owned record. Private identity is checked before every operation. */ +struct apple_dart_sleep { + void *private; + struct device *provider, *client; + struct task_struct *owner; + u64 generation; + int result; + bool active, sealed, verified; +}; + +/* + * Bind at host probe, adding a host->DART supplier link. begin/finish execute + * in the host's synchronous system-PM task. Seal follows child noirq callbacks, + * with bus mastering held and no pending transaction. The same task stops the + * streams while the port clock is still on and restores them after power loss, + * before the PCI core restores endpoint state. Abort reopens only unchanged + * state; failed recovery keeps DMA and mapping writers closed. + */ +#if IS_ENABLED(CONFIG_APPLE_DART) +int apple_dart_sleep_bind(struct device *backend, struct device *client, + struct apple_dart_sleep *sleep); +int apple_dart_sleep_begin(struct apple_dart_sleep *sleep, u64 generation); +int apple_dart_sleep_seal(struct apple_dart_sleep *sleep); +int apple_dart_sleep_abort(struct apple_dart_sleep *sleep); +int apple_dart_sleep_finish(struct apple_dart_sleep *sleep); +/* Native radio: readiness is MMIO-free; off precedes port clock removal. + * Restore is process context after parent/port clocks, with endpoint DMA held. + */ +int apple_dart_sleep_power_ready(struct apple_dart_sleep *sleep); +int apple_dart_sleep_power_off(struct apple_dart_sleep *sleep); +int apple_dart_sleep_power_restore(struct apple_dart_sleep *sleep); +#else +static inline int apple_dart_sleep_bind(struct device *backend, struct device *client, + struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_begin(struct apple_dart_sleep *sleep, u64 generation) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_seal(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_abort(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_finish(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_power_ready(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_power_off(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_power_restore(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +#endif +#endif diff --git a/include/linux/soc/apple/sleep-spin.h b/include/linux/soc/apple/sleep-spin.h new file mode 100644 index 0000000000000..056a975d64a97 --- /dev/null +++ b/include/linux/soc/apple/sleep-spin.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_SOC_APPLE_SLEEP_SPIN_H +#define _LINUX_SOC_APPLE_SLEEP_SPIN_H + +#include +#include +#include + +#ifdef CONFIG_ARM64 +#include +#endif + +/* No WFx is permitted before the platform's final sleep instruction. */ +static inline int apple_sleep_spin(void __iomem *reg, u32 mask, u32 expected, unsigned int usec) +{ +#ifdef CONFIG_ARM64 + u64 start = arch_timer_read_counter(); + u64 ticks = ((u64)arch_timer_get_rate() * usec + USEC_PER_SEC - 1) / USEC_PER_SEC; + + if (!ticks) + return -EINVAL; + /* Bound a stopped counter as well as a peripheral that never responds. */ + for (unsigned int i = 0; i < 20000000; i++) { + u64 now; + + if (reg && (readl(reg) & mask) == expected) + return 0; + now = arch_timer_read_counter(); + if (now < start) + return -EIO; + if (now - start >= ticks) + return reg ? -ETIMEDOUT : 0; + cpu_relax(); + } + return -ETIMEDOUT; +#else + return -EOPNOTSUPP; +#endif +} + +#endif diff --git a/tools/asahi/dart-sleep-power-tests.c b/tools/asahi/dart-sleep-power-tests.c new file mode 100644 index 0000000000000..92882677ad205 --- /dev/null +++ b/tools/asahi/dart-sleep-power-tests.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual native DART code; the model drops registers on controller power loss. */ +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define __iomem +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) (((~0U) << (l)) & ((~0U) >> (31 - (h)))) +#define BITS_TO_U32(n) (((n) + 31) / 32) +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +/* Single-threaded fixture: model publication without CPU barriers. */ +#define smp_store_release(p, v) (*(p) = (v)) +#define dma_wmb() ((void)0) +/* The MMIO mock records write order synchronously. */ +#define mb() ((void)0) +#define max(a, b) ((a) > (b) ? (a) : (b)) +#define FIELD_GET(mask, v) (((v) & (mask)) >> __builtin_ctz(mask)) +#define DART_PARAMS2 4 +#define DART_T8110_PARAMS2_READ_PORTS GENMASK(18, 16) +#define DART_T8110_PARAMS2_WRITE_PORTS GENMASK(22, 20) +#define DART_T8110_OUTSTANDING(n) (0x230 + 4 * (n)) +#define DART_T8110_IDLE_TIMEOUT_US 1000 +#define DART_MAX_STREAMS 32 +#define DART_MAX_TTBR 4 +#define DART_T8110 2 +#define DART_T8110_PROTECT 0x200 +#define DART_T8110_PROTECT_LOCK 0x208 +#define DART_T8110_PROTECT_TTBR_TCR BIT(0) +#define DART_T8110_V2_ERROR_MASK 0x4060 +#define DART_T8110_DISABLE_STREAMS 0xc20 +#define APPLE_PS_PROT_SLICE(n) (0x100 + 0x40 * (n)) +#define DART_TCR(d, sid) ((void)(d), 0x1000 + 4 * (sid)) +#define DART_TTBR(d, sid, idx) ((void)(d), 0x1400 + 4 * (sid) + 4 * (idx)) +struct device { + int unused; +}; +struct regmap { + u32 word; +}; +struct apple_dart_hw { + unsigned int type, ttbr_count, enable_streams; +}; +struct apple_dart { + struct device *dev; + struct apple_dart_hw *hw; + void *regs; + unsigned int num_streams, version, managed_sids; + bool sleep_closed, commands_gated; +}; +struct apple_dart_sleep { + void *private; + bool active, sealed, verified; + u64 generation; + int result; +}; +static u32 regs[0x10000 / 4]; +static bool critical; +static unsigned int writes, enabled_at, last_translation, flush_at; +static unsigned int drop_offset; +static unsigned int offset(const void *p) +{ + return (const char *)p - (const char *)regs; +} +static u32 readl(const void *p) +{ + assert(offset(p) < sizeof(regs)); + return *(const u32 *)p; +} +static void writel(u32 v, void *p) +{ + unsigned int off = offset(p); + + assert(off < sizeof(regs)); + + writes++; + if (off == drop_offset) + return; + if (off >= 0xc20 && off < 0xc24) { + regs[0xc00 / 4] &= ~v; + return; + } + if (off == 0xc00) { + regs[off / 4] |= v; + if (v) + enabled_at = writes; + return; + } + if (off == 0x4060) { + regs[off / 4] = v & 1; + return; + } + if (off >= 0x1000 && off < 0x1500) + last_translation = writes; + if (off == 0x80) + flush_at = writes; + regs[off / 4] = v; +} +#define readl_poll_timeout_atomic(p, v, cond, delay, timeout) \ + ({ \ + (void)(delay); \ + (void)(timeout); \ + int ret = -ETIMEDOUT; \ + for (unsigned int tries = 0; tries < 4; tries++) { \ + v = readl(p); \ + if (cond) { \ + ret = 0; \ + break; \ + } \ + } \ + ret; \ + }) +static int regmap_read(struct regmap *m, u32 off, u32 *v) +{ + assert(off == APPLE_PS_PROT_SLICE(28)); + *v = m->word; + return 0; +} +static int regmap_update_bits(struct regmap *m, u32 off, u32 mask, u32 v) +{ + assert(off == APPLE_PS_PROT_SLICE(28)); + m->word = (m->word & ~mask) | (v & mask); + return 0; +} +static u32 apple_dart_readl(struct apple_dart *d, u32 off) +{ + return readl(d->regs + off); +} +static void apple_dart_writel(struct apple_dart *d, u32 v, u32 off) +{ + writel(v, d->regs + off); +} +static bool apple_dart_fault_pending(struct apple_dart *d) +{ + (void)d; + return regs[0x4000 / 4] || (regs[0x704 / 4] & 0x777); +} + +static bool apple_dart_sleep_critical(void) +{ + return critical; +} +static bool irqs_disabled(void) +{ + return critical; +} +static bool in_interrupt(void) +{ + return false; +} +#include "apple-dart-power.h" +/* ACTUAL_STATE */ +static int apple_dart_sleep_valid(struct apple_dart_sleep *r) +{ + struct apple_dart_sleep_state *s = r->private; + + return !r->active || s->record != r || s->generation != r->generation ? + -ESTALE : + r->result; +} +static int apple_dart_sleep_snapshot(struct apple_dart_sleep_state *s, + bool verify) +{ + assert(verify); + for (unsigned int sid = 0; sid < s->dart->num_streams; sid++) + if (regs[(0x1000 + 4 * sid) / 4] != s->tcr[sid] || + regs[(0x1400 + 4 * sid) / 4] != s->ttbr[sid][0]) + return -EUCLEAN; + return 0; +} +#define CONFIG_ARM64 1 +#define USEC_PER_SEC 1000000 +static u64 tick; +static bool stopped_counter, backward_counter; +static u64 arch_timer_read_counter(void) +{ + if (backward_counter) + return tick--; + return stopped_counter ? tick : (tick += 1000); +} +static u32 arch_timer_get_rate(void) +{ + return 1000000; +} +static void cpu_relax(void) +{ +} +/* ACTUAL_SPIN */ +/* ACTUAL_FUNCTIONS */ +static struct apple_dart_hw hw = { DART_T8110, 1, 0xc00 }; +static struct apple_dart dart; +static struct apple_dart_sleep record; +static struct apple_dart_sleep_state state; +static struct regmap gapf; +static void fresh(void) +{ + memset(regs, 0, sizeof(regs)); + memset(&state, 0, sizeof(state)); + dart = (struct apple_dart){ .hw = &hw, + .regs = regs, + .num_streams = 19, + .version = 0x203, + .managed_sids = 6, + .sleep_closed = true }; + record = (struct apple_dart_sleep){ .private = &state, + .active = true, + .sealed = true, + .generation = 9 }; + state.dart = &dart; + state.record = &record; + state.generation = 9; + state.gapf = &gapf; + gapf.word = 0x812; + state.enabled[0] = regs[0xc00 / 4] = 6 | BIT(16) | BIT(18); + for (unsigned int sid = 0; sid < 19; sid++) { + state.tcr[sid] = regs[(0x1000 + 4 * sid) / 4] = + sid == 1 || sid == 2 ? 9 : + sid == 16 || sid == 18 ? 2 : + 0; + state.ttbr[sid][0] = regs[(0x1400 + 4 * sid) / 4] = + sid == 1 || sid == 2 ? 0x12340001 + sid * 4 : 0; + } + regs[0x210 / 4] = 0x123; + regs[0x228 / 4] = 0x42; + regs[0x22c / 4] = 0x4000000; + for (unsigned int i = 0; i < DART_POWER_TUNABLES; i++) + regs[dart_power_tunables[i].offset / 4] = + dart_power_tunables[i].mask & 0x81080809; + regs[4 / 4] = 1U << 16; + stopped_counter = backward_counter = false; + tick = 0; + critical = false; + writes = enabled_at = last_translation = flush_at = 0; + drop_offset = ~0U; +} +static void power_off(void) +{ + assert(!apple_dart_sleep_power_capture(&state)); + assert(!apple_dart_sleep_power_ready(&record)); + critical = false; + assert(!apple_dart_sleep_power_off(&record)); + assert(dart.commands_gated && dart.sleep_closed && !record.verified); + assert(!regs[0xc00 / 4] && gapf.word == 0x802); + critical = false; +} +int main(void) +{ + fresh(); + assert(apple_dart_sleep_power_ready(&record) == -ENODATA); + assert(!writes); + fresh(); + assert(!apple_dart_sleep_power_capture(&state)); + critical = true; + assert(apple_dart_sleep_power_off(&record) == -EPERM); + assert(!writes); + fresh(); + gapf.word &= ~BIT(1); + assert(apple_dart_sleep_power_capture(&state) == -EPERM); + assert(!writes); + /* Trial246: active radio DART inherited GAPF bit1, with bit4 clear. */ + for (unsigned int extra = 0; extra <= 0x800; extra += 0x800) { + fresh(); + gapf.word = BIT(1) | extra; + assert(!apple_dart_sleep_power_capture(&state)); + assert(!writes); + critical = false; + assert(!apple_dart_sleep_power_off(&record)); + critical = false; + assert(gapf.word == (BIT(1) | extra)); + memset(regs, 0, sizeof(regs)); + assert(!apple_dart_sleep_power_restore(&record)); + assert(gapf.word == (BIT(1) | extra)); + assert(record.verified && !dart.sleep_closed && + !dart.commands_gated); + } + fresh(); + dart.version = 0x204; + assert(apple_dart_sleep_power_capture(&state) == -EOPNOTSUPP); + assert(!writes); + fresh(); + power_off(); + memset(regs, 0, sizeof(regs)); + assert(!apple_dart_sleep_power_restore(&record)); + assert(record.verified && !dart.sleep_closed && !dart.commands_gated); + assert(regs[0xc00 / 4] == (6 | BIT(16) | BIT(18)) && + gapf.word == 0x812); + assert(regs[0x104 / 4] == 0 && regs[0x4060 / 4] == 0); + assert(enabled_at > flush_at && flush_at > last_translation); + assert(regs[(0x1000 + 4 * 16) / 4] == 2 && + regs[(0x1000 + 4 * 18) / 4] == 2); + assert(apple_dart_sleep_power_restore(&record) == -EPERM); + fresh(); + power_off(); + memset(regs, 0, sizeof(regs)); + drop_offset = 0x1408; + assert(apple_dart_sleep_power_restore(&record) == -EIO); + assert(!regs[0xc00 / 4] && dart.commands_gated && dart.sleep_closed && + !record.verified); + fresh(); + power_off(); + memset(regs, 0, sizeof(regs)); + regs[0x200 / 4] = regs[0x208 / 4] = 1; + assert(apple_dart_sleep_power_restore(&record) == -EUCLEAN); + assert(!regs[0xc00 / 4]); + fresh(); + power_off(); + regs[0x4000 / 4] = 1; + assert(apple_dart_sleep_power_restore(&record) == -EBUSY); + assert(!regs[0xc00 / 4]); + fresh(); + power_off(); + gapf.word &= ~BIT(1); + writes = 0; + assert(apple_dart_sleep_power_restore(&record) == -EPERM); + assert(!writes); + fresh(); + assert(!apple_dart_sleep_power_capture(&state)); + critical = false; + regs[0x80 / 4] = BIT(31); + assert(apple_dart_sleep_power_off(&record) == -ETIMEDOUT); + assert(!dart.commands_gated && dart.sleep_closed && !record.verified); + fresh(); + assert(!apple_dart_sleep_power_capture(&state)); + critical = false; + regs[0x230 / 4] = 1; + assert(apple_dart_sleep_power_off(&record) == -ETIMEDOUT); + assert(!writes); + fresh(); + assert(!apple_dart_sleep_power_capture(&state)); + critical = false; + regs[4 / 4] = 0; + assert(apple_dart_sleep_power_off(&record) == -EIO); + assert(!writes); + fresh(); + stopped_counter = true; + regs[0x80 / 4] = BIT(31); + assert(apple_sleep_spin(regs + 0x80 / 4, BIT(31), 0, 1000) == + -ETIMEDOUT); + fresh(); + tick = 10; + backward_counter = true; + assert(apple_sleep_spin(NULL, 0, 0, 1000) == -EIO); + puts("PASS: DART power loss, static streams, restore ordering and failures"); +} diff --git a/tools/asahi/dart-sleep-power-tests.py b/tools/asahi/dart-sleep-power-tests.py new file mode 100644 index 0000000000000..5071b0b2daa59 --- /dev/null +++ b/tools/asahi/dart-sleep-power-tests.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Actual native DART recovery under power loss and MMIO failures.""" + +from pathlib import Path +import re +import resource +import subprocess +import tempfile + +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/iommu/apple-dart-sleep.c").read_text() + + +def extract(name): + match = re.search( + r"^(?:static )?(?:int|void) " + name + r"\([^;]+?\{.*?\n\}", source, re.M | re.S + ) + assert match, name + return match.group() + + +state = re.search(r"struct apple_dart_sleep_state \{.*?\n\};", source, re.S).group() +names = ( + "apple_dart_sleep_gapf", + "apple_dart_sleep_power_capture", + "apple_dart_sleep_power_ready", + "apple_dart_sleep_tlb_idle", + "apple_dart_sleep_idle", + "apple_dart_sleep_power_off", + "apple_dart_sleep_power_restore", +) +actual = "\n".join(extract(name) for name in names) +fixture = Path(__file__).with_suffix(".c").read_text() +spin = (root / "include/linux/soc/apple/sleep-spin.h").read_text() +spin = re.search(r"static inline int apple_sleep_spin.*?\n\}", spin, re.S).group() +fixture = fixture.replace("/* ACTUAL_SPIN */", spin) +code = fixture.replace("/* ACTUAL_STATE */", state).replace( + "/* ACTUAL_FUNCTIONS */", actual +) +with tempfile.TemporaryDirectory(prefix="dart-power-") as tmp: + src, binary = Path(tmp) / "test.c", Path(tmp) / "test" + src.write_text(code) + subprocess.run( + [ + "cc", + "-std=gnu11", + "-Wall", + "-Wextra", + "-Werror", + "-Wno-unused-function", + "-I", + str(root / "drivers/iommu"), + str(src), + "-o", + str(binary), + ], + check=True, + ) + subprocess.run([str(binary)], check=True) diff --git a/tools/asahi/dart-sleep-snapshot-tests.c b/tools/asahi/dart-sleep-snapshot-tests.c new file mode 100644 index 0000000000000..0890911a7a995 --- /dev/null +++ b/tools/asahi/dart-sleep-snapshot-tests.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define BIT(n) (1U << (n)) +#define READ_ONCE(x) (x) +#define BITS_TO_U32(n) (((n) + 31) / 32) +#define DART_MAX_STREAMS 32 +#define DART_MAX_TTBR 4 +struct device { + int unused; +}; +struct dart_power_state { + int unused; +}; /* Power replay is covered separately. */ +struct hw_info { + unsigned int ttbr_count; + u32 lock, tcr, ttbr, ttbr_valid, tcr_4level, enable_streams; +}; +struct apple_dart { + struct device *dev; + struct hw_info *hw; + bool retained_mtp, four_level, managed_failed; + unsigned int managed_sids, num_streams, version; +}; +static u32 regs[0x5000 / 4]; +static bool irqoff, fault; +static unsigned int logs; +static bool irqs_disabled(void) +{ + return irqoff; +} +static void dev_err(struct device *d, const char *fmt, ...) +{ + (void)d; + (void)fmt; + logs++; +} +static u32 apple_dart_readl(struct apple_dart *d, u32 off) +{ + (void)d; + assert(off < sizeof(regs) && !(off & 3)); + return regs[off / 4]; +} +static bool apple_dart_fault_pending(struct apple_dart *d) +{ + (void)d; + return fault; +} + +/* ACTUAL_PRODUCTION */ +int main(void) +{ + struct hw_info hw = { .ttbr_count = 1, + .lock = DART_T8110_PROTECT, + .tcr = DART_T8110_TCR, + .ttbr = DART_T8110_TTBR, + .ttbr_valid = DART_T8110_TTBR_VALID, + .tcr_4level = DART_T8110_TCR_FOUR_LEVEL, + .enable_streams = DART_T8110_ENABLE_STREAMS }; + struct apple_dart d = { .hw = &hw, + .four_level = true, + .version = 0x0203, + .managed_sids = 6, + .num_streams = DART_MAX_STREAMS }; + struct apple_dart_sleep_state s = { .dart = &d }; + + regs[hw.enable_streams / 4] = 6; + for (unsigned int sid = 1; sid <= 2; sid++) { + regs[DART_TCR(&d, sid) / 4] = + 9; /* Captured four-level translation. */ + regs[DART_TTBR(&d, sid, 0) / 4] = 0x12340001 + 4 * sid; + } + assert(apple_dart_sleep_snapshot(&s, false) == 0); + assert(apple_dart_sleep_snapshot(&s, true) == 0); + assert(logs == 0); + /* Each retained value must stay exact, including the table-depth bit. */ + unsigned int changed[] = { + DART_TCR(&d, 1), DART_TCR(&d, 2), + DART_TTBR(&d, 1, 0), DART_TTBR(&d, 2, 0), + DART_TCR(&d, 0), DART_TTBR(&d, 0, 0), + hw.enable_streams, DART_T8110_PROTECT, + DART_T8110_PROTECT_LOCK, DART_T8110_V2_ERROR_MASK + }; + irqoff = true; + for (unsigned int i = 0; i < ARRAY_SIZE(changed); i++) { + regs[changed[i] / 4] ^= 8; + assert(apple_dart_sleep_snapshot(&s, true) == -EUCLEAN); + regs[changed[i] / 4] ^= 8; + assert(logs == 0); + } + irqoff = false; + d.four_level = false; + assert(apple_dart_sleep_snapshot(&s, false) == -EUCLEAN); + for (unsigned int sid = 1; sid <= 2; sid++) + regs[DART_TCR(&d, sid) / 4] = 1; + assert(apple_dart_sleep_snapshot(&s, false) == 0); + d.four_level = true; + assert(apple_dart_sleep_snapshot(&s, false) == 0); + /* Disabled, bypass, remap, unknown bits, and missing roots still fail. */ + unsigned int invalid[] = { + 0, 8, 2, 3, 5, 7, 0xb, 0xd, 0x81, 0x101, 0x11 + }; + + for (unsigned int sid = 1; sid <= 2; sid++) { + for (unsigned int i = 0; i < ARRAY_SIZE(invalid); i++) { + regs[DART_TCR(&d, sid) / 4] = invalid[i]; + assert(apple_dart_sleep_snapshot(&s, false) == + -EUCLEAN); + } + regs[DART_TCR(&d, sid) / 4] = 9; + regs[DART_TTBR(&d, sid, 0) / 4] &= ~DART_T8110_TTBR_VALID; + assert(apple_dart_sleep_snapshot(&s, false) == -EUCLEAN); + regs[DART_TTBR(&d, sid, 0) / 4] |= DART_T8110_TTBR_VALID; + } + assert(apple_dart_sleep_snapshot(&s, false) == 0); + for (unsigned int version = 0x0201; version <= 0x0203; version++) { + unsigned int mask = version >= 0x0202 ? + DART_T8110_V2_ERROR_MASK : + DART_T8110_ERROR_MASK; + unsigned int other = version >= 0x0202 ? + DART_T8110_ERROR_MASK : + DART_T8110_V2_ERROR_MASK; + d.version = version; + regs[mask / 4] = 1; + assert(apple_dart_sleep_snapshot(&s, false) == -EUCLEAN); + regs[mask / 4] = 0; + regs[other / 4] = ~0U; + assert(apple_dart_sleep_snapshot(&s, false) == 0); + regs[other / 4] = 0; + } + regs[DART_T8110_PROTECT / 4] = DART_T8110_PROTECT_TTBR_TCR; + assert(apple_dart_sleep_snapshot(&s, false) == -EUCLEAN); + regs[DART_T8110_PROTECT / 4] = 0; + fault = true; + assert(apple_dart_sleep_snapshot(&s, false) == -EBUSY); + fault = false; + regs[DART_T8110_TLB_CMD / 4] = DART_T8110_TLB_CMD_BUSY; + assert(apple_dart_sleep_snapshot(&s, false) == -EBUSY); + regs[DART_T8110_TLB_CMD / 4] = 0; + d.managed_failed = true; + assert(apple_dart_sleep_snapshot(&s, false) == -EUCLEAN); + d.managed_failed = false; + assert(apple_dart_sleep_snapshot(&s, false) == 0); + assert(apple_dart_sleep_snapshot(&s, true) == 0); + puts("PASS: DART table depths, corrupt roots, faults and snapshot validation"); +} diff --git a/tools/asahi/dart-sleep-snapshot-tests.py b/tools/asahi/dart-sleep-snapshot-tests.py new file mode 100644 index 0000000000000..5887b0b2046af --- /dev/null +++ b/tools/asahi/dart-sleep-snapshot-tests.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise retained DART snapshots with both supported translation depths.""" + +from pathlib import Path +import sleep_test_utils as x +import re +import resource +import subprocess +import tempfile + +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/iommu/apple-dart-sleep.c").read_text() +driver = (root / "drivers/iommu/apple-dart.c").read_text() +constants = ( + "DART_T8110_TLB_CMD", + "DART_T8110_TLB_CMD_BUSY", + "DART_T8110_PROTECT", + "DART_T8110_PROTECT_LOCK", + "DART_T8110_PROTECT_TTBR_TCR", + "DART_T8110_ERROR_MASK", + "DART_T8110_V2_ERROR_MASK", + "DART_T8110_ENABLE_STREAMS", + "DART_T8110_TCR", + "DART_T8110_TCR_TRANSLATE_ENABLE", + "DART_T8110_TCR_FOUR_LEVEL", + "DART_T8110_TTBR", + "DART_T8110_TTBR_VALID", +) +defines = "\n".join( + re.search(r"^#define\s+" + name + r"\s+[^\n]+", driver, re.M).group() + for name in constants +) +for name in ("DART_TCR", "DART_TTBR"): + defines += ( + "\n" + + re.search( + r"^#define " + name + r"\([^\n]+(?:\n[^\n]+)*?(?=\n\n)", driver, re.M + ).group() + ) +state = re.search(r"struct apple_dart_sleep_state \{.*?\n\};", source, re.S).group() +actual = "\n".join( + x.function(source, name) + for name in ("apple_dart_sleep_snapshot_error", "apple_dart_sleep_snapshot") +) +fixture, main = ( + Path(__file__).with_suffix(".c").read_text().split("/* ACTUAL_PRODUCTION */") +) +with tempfile.TemporaryDirectory(prefix="dart-snapshot-") as tmp: + src, binary = Path(tmp) / "test.c", Path(tmp) / "test" + for negative in (False, True): + code = actual + if negative: + old = "value &= ~dart->hw->tcr_4level;" + assert old in code + code = code.replace(old, "(void)value;") + src.write_text(fixture + defines + "\n" + state + "\n" + code + main) + subprocess.run( + [ + "cc", + "-std=gnu11", + "-Wall", + "-Wextra", + "-Werror", + str(src), + "-o", + str(binary), + ], + check=True, + ) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if negative: + assert result.returncode != 0, "Three-level-only admission was accepted" + print("PASS: old admission rejected by captured four-level regression") + else: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) diff --git a/tools/asahi/dart-sleep-writer-tests.c b/tools/asahi/dart-sleep-writer-tests.c new file mode 100644 index 0000000000000..8594dfb14dcb6 --- /dev/null +++ b/tools/asahi/dart-sleep-writer-tests.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define spin_lock_irqsave(lock, flags) \ + do { \ + (void)(lock); \ + (flags) = 0; \ + } while (0) +#define spin_unlock_irqrestore(lock, flags) \ + do { \ + (void)(lock); \ + (void)(flags); \ + } while (0) +struct apple_dart_sleep { + int result; +}; +struct apple_dart { + bool managed_sids, sleep_closed; + unsigned int sleep_writers; + int managed_lock; + struct apple_dart_sleep *sleep_record; +}; +/* ACTUAL_FUNCTIONS */ +int main(void) +{ + struct apple_dart_sleep s = { 0 }; + struct apple_dart d = { .managed_sids = true, .sleep_record = &s }; + + assert(!apple_dart_sleep_writer_get(&d)); + assert(!apple_dart_sleep_writer_get( + &d)); /* nested invalidation cannot deadlock */ + assert(apple_dart_sleep_close(&d) == -EBUSY && !d.sleep_closed); + apple_dart_sleep_writer_put(&d); + assert(apple_dart_sleep_close(&d) == -EBUSY); + apple_dart_sleep_writer_put(&d); + assert(!apple_dart_sleep_close(&d)); + assert(apple_dart_sleep_writer_get(&d) == -EBUSY && + s.result == -EUCLEAN && !d.sleep_writers); + d.sleep_writers = UINT_MAX; + d.sleep_closed = false; + assert(apple_dart_sleep_writer_get(&d) == -EOVERFLOW); + d = (struct apple_dart){ 0 }; + assert(!apple_dart_sleep_writer_get(&d)); + apple_dart_sleep_writer_put(&d); + assert(!d.sleep_writers); + puts("DART writer gate: nested/in-flight/closed/legacy invariants PASS"); +} diff --git a/tools/asahi/dart-sleep-writer-tests.py b/tools/asahi/dart-sleep-writer-tests.py new file mode 100644 index 0000000000000..ed01d2947441b --- /dev/null +++ b/tools/asahi/dart-sleep-writer-tests.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +from pathlib import Path +import subprocess, tempfile, re + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/iommu/apple-dart-sleep.c").read_text() +actual = [] +for name in ( + "apple_dart_sleep_writer_get", + "apple_dart_sleep_writer_put", + "apple_dart_sleep_close", +): + m = re.search( + r"^static (?:int|void) " + name + r"\([^;]+?\{.*?\n\}", source, re.M | re.S + ) + assert m, name + actual.append(m.group()) +f = ( + Path(__file__) + .with_suffix(".c") + .read_text() + .replace("/* ACTUAL_FUNCTIONS */", "\n".join(actual)) +) +with tempfile.TemporaryDirectory() as t: + p = Path(t) / "test.c" + p.write_text(f) + b = Path(t) / "test" + subprocess.run( + ["cc", "-std=gnu11", "-Wall", "-Wextra", "-Werror", str(p), "-o", str(b)], + check=True, + ) + subprocess.run([str(b)], check=True) diff --git a/tools/asahi/sleep_test_utils.py b/tools/asahi/sleep_test_utils.py new file mode 100644 index 0000000000000..246aeda3dfe08 --- /dev/null +++ b/tools/asahi/sleep_test_utils.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0-only +"""Extract production C functions and structures for sleep fault-injection tests.""" + +import re + + +def function(source, name): + # Keep offsets while hiding documentation and commented-out declarations. + code = re.sub( + r"/\*.*?\*/|//[^\n]*", + lambda match: re.sub(r"[^\n]", " ", match[0]), + source, + flags=re.S, + ) + match = re.search( + r"(?m)^[\w *\t]+\b" + re.escape(name) + r"\s*\([^;{}]*\)\s*\{", code + ) + assert match, f"missing exact function: {name}" + start = match.start() + opening = match.end() - 1 + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + "\n" + + +def structure(source, name): + start = source.index("struct " + name + " {") + end = source.index("\n};", start) + 3 + return source[start:end] + "\n"