diff --git a/MAINTAINERS b/MAINTAINERS index 0cdac3657b043..7d1c382eff471 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2778,6 +2778,8 @@ F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c F: drivers/input/misc/macsmc-input.c F: drivers/input/touchscreen/apple_z2.c +F: drivers/iommu/apple-dart-power.h +F: drivers/iommu/apple-dart-sleep.c F: drivers/iommu/apple-dart-test.c F: drivers/iommu/apple-dart.c F: drivers/iommu/io-pgtable-dart.c @@ -2824,6 +2826,8 @@ F: include/uapi/drm/asahi_drm.h F: include/uapi/linux/apple-sep.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h +F: tools/asahi/dart-sleep-*-tests.* +F: tools/asahi/sleep_test_utils.py F: tools/testing/selftests/drivers/apple/ ARM/APPLE AVE VIDEO ENCODER DRIVER diff --git a/drivers/iommu/apple-dart-power.h b/drivers/iommu/apple-dart-power.h new file mode 100644 index 0000000000000..b1c9a01720abd --- /dev/null +++ b/drivers/iommu/apple-dart-power.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* T8132 native radio DART power state. The caller owns power and excludes DMA. */ +#ifndef __APPLE_DART_POWER_STATE_H +#define __APPLE_DART_POWER_STATE_H + +#define DART_POWER_TUNABLES 6 +#define DART_POWER_TZ_WORDS 11 +#define DART_POWER_ERROR_PENDING 0x4000 +#define DART_POWER_ERROR_MASK 0x4060 + +struct dart_power_state { + u32 lock, diag, pair[2], version; + u32 tunable[DART_POWER_TUNABLES]; + u32 tz[DART_POWER_TZ_WORDS]; + bool has_tz, valid; +}; + +static const struct { + u32 offset, mask; +} dart_power_tunables[DART_POWER_TUNABLES] = { + { 0x20c, 0x80000121 }, { 0x220, 0x000f0f0f }, { 0x224, 0x00ffffff }, + { 0x300, 0x00001f31 }, { 0x308, 0x3ffffffc }, { 0x310, 0x3ffffffc }, +}; + +static const struct { + u32 offset, mask; +} dart_power_tz[DART_POWER_TZ_WORDS] = { + { 0x500, 7 }, { 0x504, 1 }, { 0x508, 0x3ffffffc }, { 0x510, 0x3ffffffc }, + { 0x518, 0x3ffffffc }, { 0x520, 0x3ffffffc }, { 0x528, 0x3ffffffc }, { 0x530, 0x3ffffffc }, + { 0x538, 0x3ffffffc }, { 0x540, 0x3ffffffc }, { 0x548, 0x3ffffffc }, +}; + +/* Only v2.2/v2.3, with one 32-stream error bitmap, are supported here. */ +static bool dart_power_fault_pending(void __iomem *regs, const struct dart_power_state *s) +{ + return readl(regs + DART_POWER_ERROR_PENDING) || + (s->version >= 0x202 && (readl(regs + 0x704) & 0x777)); +} + +/* Reserved mask bits need not read back as written; complete the posted write. */ +static void dart_power_mask_errors(void __iomem *regs, bool mask) +{ + writel(mask ? ~0U : 0, regs + DART_POWER_ERROR_MASK); + /* Finish the mask store before the ordering read and stream changes. */ + mb(); + readl(regs + DART_POWER_ERROR_MASK); +} + +/* Configuration writes complete and are checked before enabling a stream. */ +static int dart_power_write(void __iomem *regs, u32 offset, u32 value) +{ + writel(value, regs + offset); + return readl(regs + offset) == value ? 0 : -EIO; +} + +static int dart_power_config(void __iomem *regs, u32 offset, u32 mask, u32 saved, bool locked) +{ + u32 value = readl(regs + offset); + + if (!((value ^ saved) & mask)) + return 0; + if (locked) + return -EUCLEAN; + return dart_power_write(regs, offset, (value & ~mask) | (saved & mask)); +} + +static int dart_power_tlb_idle(void __iomem *regs) +{ + u32 value; + int ret; + + ret = readl_poll_timeout_atomic(regs + 0x80, value, !(value & BIT(31)), 1, 10000); + return ret ?: readl_poll_timeout_atomic(regs + 0x604, value, !(value & BIT(0)), 1, 10000); +} + +static int dart_power_flush(void __iomem *regs, bool off) +{ + int ret = dart_power_tlb_idle(regs); + + if (ret) + return ret; + writel(off ? 0xa000 : 0x8000, regs + 0x80); + return dart_power_tlb_idle(regs); +} + +#endif diff --git a/drivers/iommu/apple-dart-sleep.c b/drivers/iommu/apple-dart-sleep.c new file mode 100644 index 0000000000000..dd85d9a166625 --- /dev/null +++ b/drivers/iommu/apple-dart-sleep.c @@ -0,0 +1,585 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by apple-dart.c: J713 radio ownership across system sleep. */ +#include +#include "apple-dart-power.h" + +struct apple_dart_sleep_state { + struct apple_dart *dart; + struct apple_dart_sleep *record; + u64 generation; + u32 tcr[DART_MAX_STREAMS]; + u32 ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; + u32 enabled[BITS_TO_U32(DART_MAX_STREAMS)]; + u32 protect, protect_lock, error_mask, lock; + struct regmap *gapf; + struct dart_power_state power; + bool gapf_on; + bool power_off; +}; + +static int apple_dart_sleep_writer_get(struct apple_dart *dart) +{ + unsigned long flags; + int ret = 0; + + if (!dart->managed_sids) + return 0; + spin_lock_irqsave(&dart->managed_lock, flags); + if (dart->sleep_closed) { + if (dart->sleep_record) + WRITE_ONCE(dart->sleep_record->result, -EUCLEAN); + ret = -EBUSY; + } else if (dart->sleep_writers == UINT_MAX) { + ret = -EOVERFLOW; + } else { + dart->sleep_writers++; + } + spin_unlock_irqrestore(&dart->managed_lock, flags); + return ret; +} + +static void apple_dart_sleep_writer_put(struct apple_dart *dart) +{ + unsigned long flags; + + if (!dart->managed_sids) + return; + spin_lock_irqsave(&dart->managed_lock, flags); + dart->sleep_writers--; + spin_unlock_irqrestore(&dart->managed_lock, flags); +} + +/* Close only at zero users, so an admitted writer may nest a TLB operation. */ +static int apple_dart_sleep_close(struct apple_dart *dart) +{ + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&dart->managed_lock, flags); + if (dart->sleep_closed || dart->sleep_writers) + ret = -EBUSY; + else + dart->sleep_closed = true; + spin_unlock_irqrestore(&dart->managed_lock, flags); + return ret; +} + +/* A void unmap/free caller may release backing memory when we return. */ +static void __noreturn apple_dart_sleep_release_failed(void) +{ + panic("J713 radio: DMA state cannot be released safely"); +} + +static int apple_dart_sleep_valid(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + + if (!sleep || !sleep->private || !sleep->active || sleep->owner != current) + return -ESTALE; + s = sleep->private; + if (s->record != sleep || s->dart->sleep_record != sleep || !sleep->generation || + s->generation != sleep->generation) + return -ESTALE; + return READ_ONCE(sleep->result) ?: (READ_ONCE(s->dart->managed_failed) ? -EUCLEAN : 0); +} + +static void apple_dart_sleep_unlink(void *data) +{ + device_link_del(data); +} + +int apple_dart_sleep_bind(struct device *backend, struct device *client, + struct apple_dart_sleep *sleep) +{ + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(client); + struct device_link *link; + struct apple_dart_sleep_state *s; + struct apple_dart *dart; + int ret; + + if (!of_machine_is_compatible("apple,j713") || !cfg || !cfg->stream_maps[0].dart || + cfg->stream_maps[1].dart) + return -ENODEV; + dart = cfg->stream_maps[0].dart; + if ((!dart->managed_sids) || dart->hw->type != DART_T8110 || !READ_ONCE(dart->iommu.ready)) + return -EINVAL; + link = device_link_add(backend, dart->dev, DL_FLAG_STATELESS); + if (!link) + return -EINVAL; + ret = devm_add_action_or_reset(backend, apple_dart_sleep_unlink, link); + if (ret) + return ret; + s = devm_kzalloc(backend, sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->dart = dart; + s->record = sleep; + if (dart->managed_sids) { + struct resource res; + + if (of_address_to_resource(dart->dev->of_node, 0, &res) || + res.start != 0x490000000ULL || resource_size(&res) != 0x20000) + return -EINVAL; + s->gapf = syscon_regmap_lookup_by_compatible("apple,t8132-ps-prot"); + if (IS_ERR(s->gapf)) + return PTR_ERR(s->gapf); + } + *sleep = (struct apple_dart_sleep){ + .private = s, + .provider = dart->dev, + .client = client, + }; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_bind); + +int apple_dart_sleep_begin(struct apple_dart_sleep *sleep, u64 generation) +{ + struct apple_dart_sleep_state *s; + struct apple_dart *dart; + unsigned long flags; + int ret = 0; + + if (!sleep || !sleep->private || !generation || sleep->active || in_interrupt() || + irqs_disabled() || pm_suspend_target_state != PM_SUSPEND_MEM) + return -EINVAL; + s = sleep->private; + dart = s->dart; + if (s->record != sleep) + return -ESTALE; + spin_lock_irqsave(&dart->managed_lock, flags); + if (dart->sleep_record || dart->sleep_closed || dart->managed_failed) { + ret = -EBUSY; + } else { + s->generation = generation; + sleep->generation = generation; + sleep->owner = current; + sleep->result = 0; + sleep->sealed = false; + sleep->verified = false; + sleep->active = true; + s->power.valid = s->power_off = false; + dart->sleep_record = sleep; + } + spin_unlock_irqrestore(&dart->managed_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_begin); + +static int apple_dart_sleep_snapshot_error(struct apple_dart *dart, const char *field, + unsigned int index, u32 value, u32 expected) +{ + /* Leave diagnostics silent if a caller has disabled interrupts. */ + if (!irqs_disabled()) + dev_err(dart->dev, "sleep %s[%u]: observed=%#x expected=%#x\n", field, index, value, + expected); + return -EUCLEAN; +} + +static int apple_dart_sleep_snapshot(struct apple_dart_sleep_state *s, bool verify) +{ + struct apple_dart *dart = s->dart; + unsigned int sid, idx, count = dart->hw->ttbr_count; + u32 value; + + if (READ_ONCE(dart->managed_failed)) + return apple_dart_sleep_snapshot_error(dart, "managed-failed", 0, 1, 0); + if (dart->managed_sids && + (apple_dart_fault_pending(dart) || + (apple_dart_readl(dart, DART_T8110_TLB_CMD) & DART_T8110_TLB_CMD_BUSY))) + return -EBUSY; + value = apple_dart_readl(dart, dart->hw->lock); + if (verify && value != s->lock) + return apple_dart_sleep_snapshot_error(dart, "lock", 0, value, s->lock); + if (!verify) + s->lock = value; + for (sid = 0; sid < dart->num_streams; sid++) { + value = apple_dart_readl(dart, DART_TCR(dart, sid)); + if (verify && value != s->tcr[sid]) + return apple_dart_sleep_snapshot_error(dart, "tcr", sid, value, + s->tcr[sid]); + if (!verify) + s->tcr[sid] = value; + for (idx = 0; idx < count; idx++) { + value = apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); + if (verify && value != s->ttbr[sid][idx]) + return apple_dart_sleep_snapshot_error( + dart, "ttbr", sid * count + idx, value, s->ttbr[sid][idx]); + if (!verify) + s->ttbr[sid][idx] = value; + } + } + if (!dart->managed_sids) + return 0; + for (idx = 0; idx < BITS_TO_U32(dart->num_streams); idx++) { + value = apple_dart_readl(dart, dart->hw->enable_streams + 4 * idx); + if (verify && value != s->enabled[idx]) + return apple_dart_sleep_snapshot_error(dart, "enabled", idx, value, + s->enabled[idx]); + if (!verify) + s->enabled[idx] = value; + } + value = apple_dart_readl(dart, DART_T8110_PROTECT); + if (verify && value != s->protect) + return apple_dart_sleep_snapshot_error(dart, "protect", 0, value, s->protect); + if (!verify) + s->protect = value; + value = apple_dart_readl(dart, DART_T8110_PROTECT_LOCK); + if (verify && value != s->protect_lock) + return apple_dart_sleep_snapshot_error(dart, "protect-lock", 0, value, + s->protect_lock); + if (!verify) + s->protect_lock = value; + value = apple_dart_readl(dart, dart->version >= 0x0202 ? DART_T8110_V2_ERROR_MASK : + DART_T8110_ERROR_MASK); + if (verify && value != s->error_mask) + return apple_dart_sleep_snapshot_error(dart, "error-mask", 0, value, s->error_mask); + if (!verify) + s->error_mask = value; + if (s->error_mask) + return apple_dart_sleep_snapshot_error(dart, "error-mask-admission", 0, + s->error_mask, 0); + if (s->protect & DART_T8110_PROTECT_TTBR_TCR) + return apple_dart_sleep_snapshot_error(dart, "protect-admission", 0, s->protect, 0); + for (sid = 1; sid <= 2; sid++) { + bool live = false; + + /* Admit both supported table depths, but no bypass or remapping. */ + value = s->tcr[sid]; + if (dart->four_level) + value &= ~dart->hw->tcr_4level; + if (value != DART_T8110_TCR_TRANSLATE_ENABLE) + return apple_dart_sleep_snapshot_error(dart, "translation-admission", sid, + s->tcr[sid], + DART_T8110_TCR_TRANSLATE_ENABLE); + for (idx = 0; idx < count; idx++) + live |= !!(s->ttbr[sid][idx] & dart->hw->ttbr_valid); + if (!live) + return apple_dart_sleep_snapshot_error( + dart, "live-root", sid, s->ttbr[sid][0], dart->hw->ttbr_valid); + } + return 0; +} + +/* The native PCIe DART uses slice 28; no other slice or policy bit is changed. */ +static int apple_dart_sleep_gapf(struct apple_dart_sleep_state *s, bool on) +{ + u32 value; + int ret; + + ret = regmap_read(s->gapf, APPLE_PS_PROT_SLICE(28), &value); + if (ret) + return ret; + if (!(value & BIT(1))) + return -EPERM; + ret = regmap_update_bits(s->gapf, APPLE_PS_PROT_SLICE(28), BIT(4), on ? BIT(4) : 0); + if (!ret) + ret = regmap_read(s->gapf, APPLE_PS_PROT_SLICE(28), &value); + return ret ?: (!!(value & BIT(4)) == on ? 0 : -EIO); +} + +static int apple_dart_sleep_power_capture(struct apple_dart_sleep_state *s) +{ + struct apple_dart *dart = s->dart; + struct dart_power_state *p = &s->power; + u32 value; + int ret; + + p->valid = false; + if (!s->gapf || dart->managed_sids != (BIT(1) | BIT(2)) || dart->hw->type != DART_T8110 || + dart->hw->ttbr_count != 1 || dart->num_streams < 19 || dart->num_streams > 32 || + dart->version < 0x202 || dart->version > 0x203) + return -EOPNOTSUPP; + ret = regmap_read(s->gapf, APPLE_PS_PROT_SLICE(28), &value); + if (ret) + return ret; + if (!(value & BIT(1))) + return -EPERM; + /* A working native DART may inherit clock protection disabled. */ + s->gapf_on = !!(value & BIT(4)); + p->version = dart->version; + if (dart_power_fault_pending(dart->regs, p)) + return -EBUSY; + p->lock = s->protect_lock; + p->diag = apple_dart_readl(dart, 0x210); + p->pair[0] = apple_dart_readl(dart, 0x228); + p->pair[1] = apple_dart_readl(dart, 0x22c); + p->has_tz = apple_dart_readl(dart, 0x4) & BIT(2); + if (p->has_tz) { + if ((p->lock & GENMASK(4, 3)) != GENMASK(4, 3)) + return -EPERM; + for (unsigned int i = 0; i < DART_POWER_TZ_WORDS; i++) + p->tz[i] = apple_dart_readl(dart, dart_power_tz[i].offset) & + dart_power_tz[i].mask; + } + for (unsigned int i = 0; i < DART_POWER_TUNABLES; i++) + p->tunable[i] = apple_dart_readl(dart, dart_power_tunables[i].offset); + p->valid = true; + return 0; +} + +int apple_dart_sleep_power_ready(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + int ret = apple_dart_sleep_valid(sleep); + + if (ret) + return ret; + s = sleep->private; + if (!sleep->sealed || sleep->verified || !s->dart->sleep_closed || s->power_off) + return -EPERM; + return s->power.valid ? 0 : -ENODATA; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_power_ready); + +static int apple_dart_sleep_tlb_idle(void __iomem *regs) +{ + return apple_sleep_spin(regs + 0x80, BIT(31), 0, 10000) ?: + apple_sleep_spin(regs + 0x604, BIT(0), 0, 10000); +} + +static int apple_dart_sleep_idle(struct apple_dart_sleep_state *s) +{ + struct apple_dart *dart = s->dart; + u32 params = readl(dart->regs + DART_PARAMS2); + unsigned int ports = max(FIELD_GET(DART_T8110_PARAMS2_READ_PORTS, params), + FIELD_GET(DART_T8110_PARAMS2_WRITE_PORTS, params)); + int ret; + + if (!ports || ports > 4 || dart_power_fault_pending(dart->regs, &s->power)) + return -EIO; + ret = apple_dart_sleep_tlb_idle(dart->regs); + if (ret) + return ret; + for (unsigned int port = 0; port < ports; port++) { + ret = apple_sleep_spin(dart->regs + DART_T8110_OUTSTANDING(port), ~0U, 0, + DART_T8110_IDLE_TIMEOUT_US); + if (ret) + return ret; + } + return dart_power_fault_pending(dart->regs, &s->power) ? -EIO : 0; +} + +/* The owner has stopped its requesters; port clocks still make the DART usable. */ +int apple_dart_sleep_power_off(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + struct apple_dart *dart; + int ret = apple_dart_sleep_power_ready(sleep); + + if (ret) + return ret; + if (in_interrupt() || irqs_disabled()) + return -EPERM; + s = sleep->private; + dart = s->dart; + ret = apple_dart_sleep_snapshot(s, true); + if (ret) + return ret; + ret = apple_dart_sleep_idle(s); + if (ret) + return ret; + if (dart_power_fault_pending(dart->regs, &s->power)) + return -EBUSY; + /* The fault IRQ must not access the window after the owner drops clocks. */ + WRITE_ONCE(dart->commands_gated, true); + dart_power_mask_errors(dart->regs, true); + for (unsigned int sid = 0; sid < dart->num_streams; sid++) { + if (s->enabled[sid / 32] & BIT(sid % 32)) { + apple_dart_writel(dart, BIT(sid % 32), + DART_T8110_DISABLE_STREAMS + 4 * (sid / 32)); + /* Complete each stream-disable command before the next one. */ + mb(); + apple_dart_readl(dart, DART_T8110_DISABLE_STREAMS + 4 * (sid / 32)); + } + } + for (unsigned int i = 0; i < BITS_TO_U32(dart->num_streams); i++) + if (apple_dart_readl(dart, dart->hw->enable_streams + 4 * i)) { + ret = -EIO; + goto fail; + } + ret = apple_dart_sleep_tlb_idle(dart->regs); + if (ret) + goto fail; + writel(0xa000, dart->regs + 0x80); + ret = apple_dart_sleep_tlb_idle(dart->regs) ?: apple_dart_sleep_gapf(s, false); + if (ret) + goto fail; + s->power_off = true; + return 0; +fail: + sleep->result = ret; + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_power_off); + +/* Called after parent and port clocks, while PERST and endpoint BME stay held. */ +int apple_dart_sleep_power_restore(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + struct apple_dart *dart; + struct dart_power_state *p; + u32 protect, lock; + bool locked; + int ret = apple_dart_sleep_valid(sleep); + + if (ret) + return ret; + s = sleep->private; + dart = s->dart; + p = &s->power; + if (in_interrupt() || irqs_disabled() || !sleep->sealed || sleep->verified || + !s->power_off || !p->valid || !READ_ONCE(dart->commands_gated) || + !READ_ONCE(dart->sleep_closed)) + return -EPERM; + ret = apple_dart_sleep_gapf(s, s->gapf_on); + if (ret) + goto fail; + for (unsigned int i = 0; i < BITS_TO_U32(dart->num_streams); i++) + if (apple_dart_readl(dart, dart->hw->enable_streams + 4 * i)) { + ret = -EBUSY; + goto fail; + } + if (dart_power_fault_pending(dart->regs, p)) { + ret = -EBUSY; + goto fail; + } + protect = apple_dart_readl(dart, DART_T8110_PROTECT); + lock = apple_dart_readl(dart, DART_T8110_PROTECT_LOCK); + locked = protect & lock & BIT(0); + if ((lock & GENMASK(4, 3)) != (p->lock & GENMASK(4, 3)) || + !!(apple_dart_readl(dart, 0x4) & BIT(2)) != p->has_tz) { + ret = -EUCLEAN; + goto fail; + } + dart_power_mask_errors(dart->regs, true); + ret = dart_power_write(dart->regs, 0x210, p->diag); + if (ret) + goto fail; + if (p->has_tz) + for (unsigned int i = 0; i < DART_POWER_TZ_WORDS; i++) + if ((apple_dart_readl(dart, dart_power_tz[i].offset) & + dart_power_tz[i].mask) != p->tz[i]) { + ret = -EUCLEAN; + goto fail; + } + for (unsigned int i = 0; i < DART_POWER_TUNABLES; i++) { + bool apf_locked = dart_power_tunables[i].offset >= 0x300 && + (protect & lock & BIT(2)); + + ret = dart_power_config(dart->regs, dart_power_tunables[i].offset, + dart_power_tunables[i].mask, p->tunable[i], + locked || apf_locked); + if (ret) + goto fail; + } + for (unsigned int sid = 0; sid < dart->num_streams; sid++) { + ret = dart_power_config(dart->regs, DART_TCR(dart, sid), ~0U, + s->tcr[sid], locked) ?: + dart_power_config(dart->regs, DART_TTBR(dart, sid, 0), ~0U, + s->ttbr[sid][0], locked); + if (ret) + goto fail; + } + ret = dart_power_flush(dart->regs, false) ?: + dart_power_config(dart->regs, 0x228, ~0U, p->pair[0], locked) ?: + dart_power_config(dart->regs, 0x22c, ~0U, p->pair[1], locked); + if (ret) + goto fail; + dart_power_mask_errors(dart->regs, false); + for (unsigned int i = 0; i < BITS_TO_U32(dart->num_streams); i++) { + ret = dart_power_write(dart->regs, dart->hw->enable_streams + 4 * i, s->enabled[i]); + if (ret) + goto fail; + } + dma_wmb(); + sleep->verified = true; + /* Publish the complete translation context before IRQ or mapping users. */ + smp_store_release(&dart->commands_gated, false); + /* Mapping writers acquire the same completed restore through this gate. */ + smp_store_release(&dart->sleep_closed, false); + return 0; +fail: + sleep->result = ret; + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_power_restore); + +int apple_dart_sleep_seal(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + int ret = apple_dart_sleep_valid(sleep); + + if (ret) + return ret; + if (irqs_disabled() || sleep->sealed) + return -EPERM; + s = sleep->private; + ret = apple_dart_sleep_close(s->dart); + if (ret) + return ret; + /* No table publication/invalidation is still in flight beyond this gate. */ + dma_wmb(); + ret = apple_dart_sleep_snapshot(s, false); + if (!ret && s->dart->managed_sids) + ret = apple_dart_sleep_power_capture(s); + if (ret) { + /* Corrupt live roots must not regain DMA even on pre-entry unwind. */ + if (ret == -EUCLEAN) { + sleep->result = -EUCLEAN; + return ret; + } + /* Publish cancellation before allowing table writers again. */ + smp_store_release(&s->dart->sleep_closed, false); + return ret; + } + sleep->sealed = true; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_seal); + +int apple_dart_sleep_abort(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + int ret = apple_dart_sleep_valid(sleep); + + if (ret) + return ret; + if (irqs_disabled() || sleep->verified) + return -EPERM; + s = sleep->private; + if (s->power_off || READ_ONCE(s->dart->commands_gated)) + return -EPERM; + if (sleep->sealed) { + ret = apple_dart_sleep_snapshot(s, true); + if (ret) { + sleep->result = -EUCLEAN; + return -EUCLEAN; + } + } + /* The saved translation state was checked before reopening writers. */ + smp_store_release(&s->dart->sleep_closed, false); + sleep->sealed = false; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_abort); + +int apple_dart_sleep_finish(struct apple_dart_sleep *sleep) +{ + struct apple_dart_sleep_state *s; + unsigned long flags; + int ret = apple_dart_sleep_valid(sleep); + + if (ret) + return ret; + s = sleep->private; + if (irqs_disabled() || READ_ONCE(s->dart->sleep_closed)) + return -EPERM; + spin_lock_irqsave(&s->dart->managed_lock, flags); + s->dart->sleep_record = NULL; + sleep->active = false; + sleep->owner = NULL; + spin_unlock_irqrestore(&s->dart->managed_lock, flags); + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_sleep_finish); diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 87d2ccef7f2b7..c60c45477b9ec 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -154,6 +155,7 @@ #define DART_T8110_ERROR_FLAG BIT(31) #define DART_T8110_ERROR_MASK 0x104 +#define DART_T8110_V2_ERROR_MASK 0x4060 #define DART_T8110_ERROR_READ_FAULT BIT(5) #define DART_T8110_ERROR_WRITE_FAULT BIT(4) @@ -457,6 +459,9 @@ struct apple_dart { bool managed_failed; /* Serialize owned TCR publication against IRQ quarantine. */ spinlock_t managed_lock; + struct apple_dart_sleep *sleep_record; + unsigned int sleep_writers; + bool sleep_closed; DECLARE_BITMAP(remapped, DART_MAX_STREAMS); u8 remap_target[DART_MAX_STREAMS]; @@ -493,6 +498,10 @@ struct apple_dart { int locked_retire_err[DART_MAX_STREAMS]; }; +static int apple_dart_sleep_writer_get(struct apple_dart *dart); +static void apple_dart_sleep_writer_put(struct apple_dart *dart); +static void __noreturn apple_dart_sleep_release_failed(void); + #ifndef MODULE static struct kexec_blocker apple_dart_radio_kexec = { .users = ATOMIC_INIT(0), @@ -2336,8 +2345,11 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, err = err ?: ret; continue; } + if (apple_dart_sleep_writer_get(stream.dart)) + apple_dart_sleep_release_failed(); ret = pm_runtime_resume_and_get(stream.dart->dev); if (ret < 0) { + apple_dart_sleep_writer_put(stream.dart); dev_err_ratelimited(stream.dart->dev, "failed to power DART for TLB synchronization: %d\n", ret); err = err ?: ret; @@ -2364,6 +2376,7 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, if (ret && stream.dart->managed_sids) apple_dart_managed_quarantine(stream.dart); pm_runtime_put(stream.dart->dev); + apple_dart_sleep_writer_put(stream.dart); err = err ?: ret; } return err; @@ -2566,7 +2579,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, return apple_dart_inherited_phys(dart_domain, address); } -static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, +static int apple_dart_map_pages_unguarded(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) @@ -2600,7 +2613,26 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, mapped); } -static size_t apple_dart_unmap_pages(struct iommu_domain *domain, +static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct apple_dart_domain *cfg = to_dart_domain(domain); + struct apple_dart *dart = cfg->stream_maps[0].dart; + int result; + int gate; + + gate = dart ? apple_dart_sleep_writer_get(dart) : 0; + if (gate) + return gate; + result = apple_dart_map_pages_unguarded(domain, iova, paddr, pgsize, pgcount, + prot, gfp, mapped); + if (dart) + apple_dart_sleep_writer_put(dart); + return result; +} + +static size_t apple_dart_unmap_pages_unguarded(struct iommu_domain *domain, unsigned long iova, size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) @@ -2616,6 +2648,23 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain, gather); } +static size_t apple_dart_unmap_pages(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct apple_dart_domain *cfg = to_dart_domain(domain); + struct apple_dart *dart = cfg->stream_maps[0].dart; + size_t result; + int gate; + + gate = dart ? apple_dart_sleep_writer_get(dart) : 0; + if (gate) + apple_dart_sleep_release_failed(); + result = apple_dart_unmap_pages_unguarded(domain, iova, pgsize, pgcount, gather); + if (dart) + apple_dart_sleep_writer_put(dart); + return result; +} + static int apple_dart_setup_translation(struct apple_dart_domain *domain, struct apple_dart_stream_map *stream_map) @@ -2890,7 +2939,7 @@ static int apple_dart_domain_add_streams(struct apple_dart_domain *domain, true); } -static int apple_dart_attach_dev_paging(struct iommu_domain *domain, +static int apple_dart_attach_dev_paging_unguarded(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) { @@ -2950,7 +2999,24 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain, return ret; } -static int apple_dart_attach_dev_identity(struct iommu_domain *domain, +static int apple_dart_attach_dev_paging(struct iommu_domain *domain, struct device *dev, + struct iommu_domain *old) +{ + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + struct apple_dart *dart = cfg->stream_maps[0].dart; + int result; + int gate; + + gate = dart ? apple_dart_sleep_writer_get(dart) : 0; + if (gate) + return gate; + result = apple_dart_attach_dev_paging_unguarded(domain, dev, old); + if (dart) + apple_dart_sleep_writer_put(dart); + return result; +} + +static int apple_dart_attach_dev_identity_unguarded(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) { @@ -2976,6 +3042,23 @@ static int apple_dart_attach_dev_identity(struct iommu_domain *domain, return 0; } +static int apple_dart_attach_dev_identity(struct iommu_domain *domain, struct device *dev, + struct iommu_domain *old) +{ + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + struct apple_dart *dart = cfg->stream_maps[0].dart; + int result; + int gate; + + gate = dart ? apple_dart_sleep_writer_get(dart) : 0; + if (gate) + return gate; + result = apple_dart_attach_dev_identity_unguarded(domain, dev, old); + if (dart) + apple_dart_sleep_writer_put(dart); + return result; +} + static const struct iommu_domain_ops apple_dart_identity_ops = { .attach_dev = apple_dart_attach_dev_identity, }; @@ -2985,7 +3068,7 @@ static struct iommu_domain apple_dart_identity_domain = { .ops = &apple_dart_identity_ops, }; -static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, +static int apple_dart_attach_dev_blocked_unguarded(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) { @@ -3007,6 +3090,23 @@ static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, return 0; } +static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, struct device *dev, + struct iommu_domain *old) +{ + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + struct apple_dart *dart = cfg->stream_maps[0].dart; + int result; + int gate; + + gate = dart ? apple_dart_sleep_writer_get(dart) : 0; + if (gate) + return gate; + result = apple_dart_attach_dev_blocked_unguarded(domain, dev, old); + if (dart) + apple_dart_sleep_writer_put(dart); + return result; +} + static const struct iommu_domain_ops apple_dart_blocked_ops = { .attach_dev = apple_dart_attach_dev_blocked, }; @@ -3055,6 +3155,8 @@ static void apple_dart_release_device(struct device *dev) struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); for_each_stream_map(j, cfg, stream_map) { + if (READ_ONCE(stream_map->dart->sleep_record)) + apple_dart_sleep_release_failed(); /* Borrowed display roots stay exactly as the DCP left them. */ if (!stream_map->dart->locked || apple_dart_has_borrowed(stream_map->dart)) continue; @@ -3128,7 +3230,7 @@ static void apple_dart_domain_free(struct iommu_domain *domain) return; } for_each_stream_map(i, dart_domain, map) { - if (map->dart->managed_sids) + if (map->dart->managed_sids || READ_ONCE(map->dart->sleep_record)) return; if (!map->dart->locked) continue; @@ -5511,6 +5613,8 @@ static const struct apple_dart_hw apple_dart_hw_t8110 = { .ttbr_count = 1, }; +#include "apple-dart-sleep.c" + static void apple_dart_save_state(struct apple_dart *dart) { unsigned int sid, idx; @@ -5650,6 +5754,10 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); + /* Native radio owner restores before the PCI functions resume. */ + if (dart->managed_sids && READ_ONCE(dart->sleep_record)) + return 0; + dart->tunnel_state_restored = false; /* Its consumer saved this already and restores it: see apple_dart_sleep_prepare(). */ if (dart->sleep_saved) @@ -5801,6 +5909,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) unsigned int sid, idx; int ret; + /* Native radio owner restores before the PCI functions resume. */ + if (dart->managed_sids && READ_ONCE(dart->sleep_record)) + return 0; + /* No reset or restore: see apple_dart_suspend(). */ if (dart->cold_ane || dart->sleep_saved) return 0; @@ -5904,6 +6016,10 @@ static __maybe_unused int apple_dart_resume_noirq(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); int ret; + /* Its host restores every register after the native port clocks return. */ + if (dart->managed_sids && READ_ONCE(dart->sleep_record)) + return pm_runtime_force_resume(dev); + /* * After a real sleep a DART clocked by its host's port reads nothing until the host * sets the port up again. Gate it, so that nothing touches it, until then. diff --git a/include/linux/soc/apple/dart-sleep.h b/include/linux/soc/apple/dart-sleep.h new file mode 100644 index 0000000000000..6e41b324b4dae --- /dev/null +++ b/include/linux/soc/apple/dart-sleep.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_SOC_APPLE_DART_SLEEP_H +#define _LINUX_SOC_APPLE_DART_SLEEP_H + +#include +#include +#include + +struct device; +struct task_struct; + +/* Caller-owned record. Private identity is checked before every operation. */ +struct apple_dart_sleep { + void *private; + struct device *provider, *client; + struct task_struct *owner; + u64 generation; + int result; + bool active, sealed, verified; +}; + +/* + * Bind at host probe, adding a host->DART supplier link. begin/finish execute + * in the host's synchronous system-PM task. Seal follows child noirq callbacks, + * with bus mastering held and no pending transaction. The same task stops the + * streams while the port clock is still on and restores them after power loss, + * before the PCI core restores endpoint state. Abort reopens only unchanged + * state; failed recovery keeps DMA and mapping writers closed. + */ +#if IS_ENABLED(CONFIG_APPLE_DART) +int apple_dart_sleep_bind(struct device *backend, struct device *client, + struct apple_dart_sleep *sleep); +int apple_dart_sleep_begin(struct apple_dart_sleep *sleep, u64 generation); +int apple_dart_sleep_seal(struct apple_dart_sleep *sleep); +int apple_dart_sleep_abort(struct apple_dart_sleep *sleep); +int apple_dart_sleep_finish(struct apple_dart_sleep *sleep); +/* Native radio: readiness is MMIO-free; off precedes port clock removal. + * Restore is process context after parent/port clocks, with endpoint DMA held. + */ +int apple_dart_sleep_power_ready(struct apple_dart_sleep *sleep); +int apple_dart_sleep_power_off(struct apple_dart_sleep *sleep); +int apple_dart_sleep_power_restore(struct apple_dart_sleep *sleep); +#else +static inline int apple_dart_sleep_bind(struct device *backend, struct device *client, + struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_begin(struct apple_dart_sleep *sleep, u64 generation) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_seal(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_abort(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_finish(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_power_ready(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_power_off(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_sleep_power_restore(struct apple_dart_sleep *sleep) +{ + return -EOPNOTSUPP; +} +#endif +#endif diff --git a/include/linux/soc/apple/sleep-spin.h b/include/linux/soc/apple/sleep-spin.h new file mode 100644 index 0000000000000..056a975d64a97 --- /dev/null +++ b/include/linux/soc/apple/sleep-spin.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_SOC_APPLE_SLEEP_SPIN_H +#define _LINUX_SOC_APPLE_SLEEP_SPIN_H + +#include +#include +#include + +#ifdef CONFIG_ARM64 +#include +#endif + +/* No WFx is permitted before the platform's final sleep instruction. */ +static inline int apple_sleep_spin(void __iomem *reg, u32 mask, u32 expected, unsigned int usec) +{ +#ifdef CONFIG_ARM64 + u64 start = arch_timer_read_counter(); + u64 ticks = ((u64)arch_timer_get_rate() * usec + USEC_PER_SEC - 1) / USEC_PER_SEC; + + if (!ticks) + return -EINVAL; + /* Bound a stopped counter as well as a peripheral that never responds. */ + for (unsigned int i = 0; i < 20000000; i++) { + u64 now; + + if (reg && (readl(reg) & mask) == expected) + return 0; + now = arch_timer_read_counter(); + if (now < start) + return -EIO; + if (now - start >= ticks) + return reg ? -ETIMEDOUT : 0; + cpu_relax(); + } + return -ETIMEDOUT; +#else + return -EOPNOTSUPP; +#endif +} + +#endif diff --git a/tools/asahi/dart-sleep-power-tests.c b/tools/asahi/dart-sleep-power-tests.c new file mode 100644 index 0000000000000..92882677ad205 --- /dev/null +++ b/tools/asahi/dart-sleep-power-tests.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual native DART code; the model drops registers on controller power loss. */ +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define __iomem +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) (((~0U) << (l)) & ((~0U) >> (31 - (h)))) +#define BITS_TO_U32(n) (((n) + 31) / 32) +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +/* Single-threaded fixture: model publication without CPU barriers. */ +#define smp_store_release(p, v) (*(p) = (v)) +#define dma_wmb() ((void)0) +/* The MMIO mock records write order synchronously. */ +#define mb() ((void)0) +#define max(a, b) ((a) > (b) ? (a) : (b)) +#define FIELD_GET(mask, v) (((v) & (mask)) >> __builtin_ctz(mask)) +#define DART_PARAMS2 4 +#define DART_T8110_PARAMS2_READ_PORTS GENMASK(18, 16) +#define DART_T8110_PARAMS2_WRITE_PORTS GENMASK(22, 20) +#define DART_T8110_OUTSTANDING(n) (0x230 + 4 * (n)) +#define DART_T8110_IDLE_TIMEOUT_US 1000 +#define DART_MAX_STREAMS 32 +#define DART_MAX_TTBR 4 +#define DART_T8110 2 +#define DART_T8110_PROTECT 0x200 +#define DART_T8110_PROTECT_LOCK 0x208 +#define DART_T8110_PROTECT_TTBR_TCR BIT(0) +#define DART_T8110_V2_ERROR_MASK 0x4060 +#define DART_T8110_DISABLE_STREAMS 0xc20 +#define APPLE_PS_PROT_SLICE(n) (0x100 + 0x40 * (n)) +#define DART_TCR(d, sid) ((void)(d), 0x1000 + 4 * (sid)) +#define DART_TTBR(d, sid, idx) ((void)(d), 0x1400 + 4 * (sid) + 4 * (idx)) +struct device { + int unused; +}; +struct regmap { + u32 word; +}; +struct apple_dart_hw { + unsigned int type, ttbr_count, enable_streams; +}; +struct apple_dart { + struct device *dev; + struct apple_dart_hw *hw; + void *regs; + unsigned int num_streams, version, managed_sids; + bool sleep_closed, commands_gated; +}; +struct apple_dart_sleep { + void *private; + bool active, sealed, verified; + u64 generation; + int result; +}; +static u32 regs[0x10000 / 4]; +static bool critical; +static unsigned int writes, enabled_at, last_translation, flush_at; +static unsigned int drop_offset; +static unsigned int offset(const void *p) +{ + return (const char *)p - (const char *)regs; +} +static u32 readl(const void *p) +{ + assert(offset(p) < sizeof(regs)); + return *(const u32 *)p; +} +static void writel(u32 v, void *p) +{ + unsigned int off = offset(p); + + assert(off < sizeof(regs)); + + writes++; + if (off == drop_offset) + return; + if (off >= 0xc20 && off < 0xc24) { + regs[0xc00 / 4] &= ~v; + return; + } + if (off == 0xc00) { + regs[off / 4] |= v; + if (v) + enabled_at = writes; + return; + } + if (off == 0x4060) { + regs[off / 4] = v & 1; + return; + } + if (off >= 0x1000 && off < 0x1500) + last_translation = writes; + if (off == 0x80) + flush_at = writes; + regs[off / 4] = v; +} +#define readl_poll_timeout_atomic(p, v, cond, delay, timeout) \ + ({ \ + (void)(delay); \ + (void)(timeout); \ + int ret = -ETIMEDOUT; \ + for (unsigned int tries = 0; tries < 4; tries++) { \ + v = readl(p); \ + if (cond) { \ + ret = 0; \ + break; \ + } \ + } \ + ret; \ + }) +static int regmap_read(struct regmap *m, u32 off, u32 *v) +{ + assert(off == APPLE_PS_PROT_SLICE(28)); + *v = m->word; + return 0; +} +static int regmap_update_bits(struct regmap *m, u32 off, u32 mask, u32 v) +{ + assert(off == APPLE_PS_PROT_SLICE(28)); + m->word = (m->word & ~mask) | (v & mask); + return 0; +} +static u32 apple_dart_readl(struct apple_dart *d, u32 off) +{ + return readl(d->regs + off); +} +static void apple_dart_writel(struct apple_dart *d, u32 v, u32 off) +{ + writel(v, d->regs + off); +} +static bool apple_dart_fault_pending(struct apple_dart *d) +{ + (void)d; + return regs[0x4000 / 4] || (regs[0x704 / 4] & 0x777); +} + +static bool apple_dart_sleep_critical(void) +{ + return critical; +} +static bool irqs_disabled(void) +{ + return critical; +} +static bool in_interrupt(void) +{ + return false; +} +#include "apple-dart-power.h" +/* ACTUAL_STATE */ +static int apple_dart_sleep_valid(struct apple_dart_sleep *r) +{ + struct apple_dart_sleep_state *s = r->private; + + return !r->active || s->record != r || s->generation != r->generation ? + -ESTALE : + r->result; +} +static int apple_dart_sleep_snapshot(struct apple_dart_sleep_state *s, + bool verify) +{ + assert(verify); + for (unsigned int sid = 0; sid < s->dart->num_streams; sid++) + if (regs[(0x1000 + 4 * sid) / 4] != s->tcr[sid] || + regs[(0x1400 + 4 * sid) / 4] != s->ttbr[sid][0]) + return -EUCLEAN; + return 0; +} +#define CONFIG_ARM64 1 +#define USEC_PER_SEC 1000000 +static u64 tick; +static bool stopped_counter, backward_counter; +static u64 arch_timer_read_counter(void) +{ + if (backward_counter) + return tick--; + return stopped_counter ? tick : (tick += 1000); +} +static u32 arch_timer_get_rate(void) +{ + return 1000000; +} +static void cpu_relax(void) +{ +} +/* ACTUAL_SPIN */ +/* ACTUAL_FUNCTIONS */ +static struct apple_dart_hw hw = { DART_T8110, 1, 0xc00 }; +static struct apple_dart dart; +static struct apple_dart_sleep record; +static struct apple_dart_sleep_state state; +static struct regmap gapf; +static void fresh(void) +{ + memset(regs, 0, sizeof(regs)); + memset(&state, 0, sizeof(state)); + dart = (struct apple_dart){ .hw = &hw, + .regs = regs, + .num_streams = 19, + .version = 0x203, + .managed_sids = 6, + .sleep_closed = true }; + record = (struct apple_dart_sleep){ .private = &state, + .active = true, + .sealed = true, + .generation = 9 }; + state.dart = &dart; + state.record = &record; + state.generation = 9; + state.gapf = &gapf; + gapf.word = 0x812; + state.enabled[0] = regs[0xc00 / 4] = 6 | BIT(16) | BIT(18); + for (unsigned int sid = 0; sid < 19; sid++) { + state.tcr[sid] = regs[(0x1000 + 4 * sid) / 4] = + sid == 1 || sid == 2 ? 9 : + sid == 16 || sid == 18 ? 2 : + 0; + state.ttbr[sid][0] = regs[(0x1400 + 4 * sid) / 4] = + sid == 1 || sid == 2 ? 0x12340001 + sid * 4 : 0; + } + regs[0x210 / 4] = 0x123; + regs[0x228 / 4] = 0x42; + regs[0x22c / 4] = 0x4000000; + for (unsigned int i = 0; i < DART_POWER_TUNABLES; i++) + regs[dart_power_tunables[i].offset / 4] = + dart_power_tunables[i].mask & 0x81080809; + regs[4 / 4] = 1U << 16; + stopped_counter = backward_counter = false; + tick = 0; + critical = false; + writes = enabled_at = last_translation = flush_at = 0; + drop_offset = ~0U; +} +static void power_off(void) +{ + assert(!apple_dart_sleep_power_capture(&state)); + assert(!apple_dart_sleep_power_ready(&record)); + critical = false; + assert(!apple_dart_sleep_power_off(&record)); + assert(dart.commands_gated && dart.sleep_closed && !record.verified); + assert(!regs[0xc00 / 4] && gapf.word == 0x802); + critical = false; +} +int main(void) +{ + fresh(); + assert(apple_dart_sleep_power_ready(&record) == -ENODATA); + assert(!writes); + fresh(); + assert(!apple_dart_sleep_power_capture(&state)); + critical = true; + assert(apple_dart_sleep_power_off(&record) == -EPERM); + assert(!writes); + fresh(); + gapf.word &= ~BIT(1); + assert(apple_dart_sleep_power_capture(&state) == -EPERM); + assert(!writes); + /* Trial246: active radio DART inherited GAPF bit1, with bit4 clear. */ + for (unsigned int extra = 0; extra <= 0x800; extra += 0x800) { + fresh(); + gapf.word = BIT(1) | extra; + assert(!apple_dart_sleep_power_capture(&state)); + assert(!writes); + critical = false; + assert(!apple_dart_sleep_power_off(&record)); + critical = false; + assert(gapf.word == (BIT(1) | extra)); + memset(regs, 0, sizeof(regs)); + assert(!apple_dart_sleep_power_restore(&record)); + assert(gapf.word == (BIT(1) | extra)); + assert(record.verified && !dart.sleep_closed && + !dart.commands_gated); + } + fresh(); + dart.version = 0x204; + assert(apple_dart_sleep_power_capture(&state) == -EOPNOTSUPP); + assert(!writes); + fresh(); + power_off(); + memset(regs, 0, sizeof(regs)); + assert(!apple_dart_sleep_power_restore(&record)); + assert(record.verified && !dart.sleep_closed && !dart.commands_gated); + assert(regs[0xc00 / 4] == (6 | BIT(16) | BIT(18)) && + gapf.word == 0x812); + assert(regs[0x104 / 4] == 0 && regs[0x4060 / 4] == 0); + assert(enabled_at > flush_at && flush_at > last_translation); + assert(regs[(0x1000 + 4 * 16) / 4] == 2 && + regs[(0x1000 + 4 * 18) / 4] == 2); + assert(apple_dart_sleep_power_restore(&record) == -EPERM); + fresh(); + power_off(); + memset(regs, 0, sizeof(regs)); + drop_offset = 0x1408; + assert(apple_dart_sleep_power_restore(&record) == -EIO); + assert(!regs[0xc00 / 4] && dart.commands_gated && dart.sleep_closed && + !record.verified); + fresh(); + power_off(); + memset(regs, 0, sizeof(regs)); + regs[0x200 / 4] = regs[0x208 / 4] = 1; + assert(apple_dart_sleep_power_restore(&record) == -EUCLEAN); + assert(!regs[0xc00 / 4]); + fresh(); + power_off(); + regs[0x4000 / 4] = 1; + assert(apple_dart_sleep_power_restore(&record) == -EBUSY); + assert(!regs[0xc00 / 4]); + fresh(); + power_off(); + gapf.word &= ~BIT(1); + writes = 0; + assert(apple_dart_sleep_power_restore(&record) == -EPERM); + assert(!writes); + fresh(); + assert(!apple_dart_sleep_power_capture(&state)); + critical = false; + regs[0x80 / 4] = BIT(31); + assert(apple_dart_sleep_power_off(&record) == -ETIMEDOUT); + assert(!dart.commands_gated && dart.sleep_closed && !record.verified); + fresh(); + assert(!apple_dart_sleep_power_capture(&state)); + critical = false; + regs[0x230 / 4] = 1; + assert(apple_dart_sleep_power_off(&record) == -ETIMEDOUT); + assert(!writes); + fresh(); + assert(!apple_dart_sleep_power_capture(&state)); + critical = false; + regs[4 / 4] = 0; + assert(apple_dart_sleep_power_off(&record) == -EIO); + assert(!writes); + fresh(); + stopped_counter = true; + regs[0x80 / 4] = BIT(31); + assert(apple_sleep_spin(regs + 0x80 / 4, BIT(31), 0, 1000) == + -ETIMEDOUT); + fresh(); + tick = 10; + backward_counter = true; + assert(apple_sleep_spin(NULL, 0, 0, 1000) == -EIO); + puts("PASS: DART power loss, static streams, restore ordering and failures"); +} diff --git a/tools/asahi/dart-sleep-power-tests.py b/tools/asahi/dart-sleep-power-tests.py new file mode 100644 index 0000000000000..5071b0b2daa59 --- /dev/null +++ b/tools/asahi/dart-sleep-power-tests.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Actual native DART recovery under power loss and MMIO failures.""" + +from pathlib import Path +import re +import resource +import subprocess +import tempfile + +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/iommu/apple-dart-sleep.c").read_text() + + +def extract(name): + match = re.search( + r"^(?:static )?(?:int|void) " + name + r"\([^;]+?\{.*?\n\}", source, re.M | re.S + ) + assert match, name + return match.group() + + +state = re.search(r"struct apple_dart_sleep_state \{.*?\n\};", source, re.S).group() +names = ( + "apple_dart_sleep_gapf", + "apple_dart_sleep_power_capture", + "apple_dart_sleep_power_ready", + "apple_dart_sleep_tlb_idle", + "apple_dart_sleep_idle", + "apple_dart_sleep_power_off", + "apple_dart_sleep_power_restore", +) +actual = "\n".join(extract(name) for name in names) +fixture = Path(__file__).with_suffix(".c").read_text() +spin = (root / "include/linux/soc/apple/sleep-spin.h").read_text() +spin = re.search(r"static inline int apple_sleep_spin.*?\n\}", spin, re.S).group() +fixture = fixture.replace("/* ACTUAL_SPIN */", spin) +code = fixture.replace("/* ACTUAL_STATE */", state).replace( + "/* ACTUAL_FUNCTIONS */", actual +) +with tempfile.TemporaryDirectory(prefix="dart-power-") as tmp: + src, binary = Path(tmp) / "test.c", Path(tmp) / "test" + src.write_text(code) + subprocess.run( + [ + "cc", + "-std=gnu11", + "-Wall", + "-Wextra", + "-Werror", + "-Wno-unused-function", + "-I", + str(root / "drivers/iommu"), + str(src), + "-o", + str(binary), + ], + check=True, + ) + subprocess.run([str(binary)], check=True) diff --git a/tools/asahi/dart-sleep-snapshot-tests.c b/tools/asahi/dart-sleep-snapshot-tests.c new file mode 100644 index 0000000000000..0890911a7a995 --- /dev/null +++ b/tools/asahi/dart-sleep-snapshot-tests.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define BIT(n) (1U << (n)) +#define READ_ONCE(x) (x) +#define BITS_TO_U32(n) (((n) + 31) / 32) +#define DART_MAX_STREAMS 32 +#define DART_MAX_TTBR 4 +struct device { + int unused; +}; +struct dart_power_state { + int unused; +}; /* Power replay is covered separately. */ +struct hw_info { + unsigned int ttbr_count; + u32 lock, tcr, ttbr, ttbr_valid, tcr_4level, enable_streams; +}; +struct apple_dart { + struct device *dev; + struct hw_info *hw; + bool retained_mtp, four_level, managed_failed; + unsigned int managed_sids, num_streams, version; +}; +static u32 regs[0x5000 / 4]; +static bool irqoff, fault; +static unsigned int logs; +static bool irqs_disabled(void) +{ + return irqoff; +} +static void dev_err(struct device *d, const char *fmt, ...) +{ + (void)d; + (void)fmt; + logs++; +} +static u32 apple_dart_readl(struct apple_dart *d, u32 off) +{ + (void)d; + assert(off < sizeof(regs) && !(off & 3)); + return regs[off / 4]; +} +static bool apple_dart_fault_pending(struct apple_dart *d) +{ + (void)d; + return fault; +} + +/* ACTUAL_PRODUCTION */ +int main(void) +{ + struct hw_info hw = { .ttbr_count = 1, + .lock = DART_T8110_PROTECT, + .tcr = DART_T8110_TCR, + .ttbr = DART_T8110_TTBR, + .ttbr_valid = DART_T8110_TTBR_VALID, + .tcr_4level = DART_T8110_TCR_FOUR_LEVEL, + .enable_streams = DART_T8110_ENABLE_STREAMS }; + struct apple_dart d = { .hw = &hw, + .four_level = true, + .version = 0x0203, + .managed_sids = 6, + .num_streams = DART_MAX_STREAMS }; + struct apple_dart_sleep_state s = { .dart = &d }; + + regs[hw.enable_streams / 4] = 6; + for (unsigned int sid = 1; sid <= 2; sid++) { + regs[DART_TCR(&d, sid) / 4] = + 9; /* Captured four-level translation. */ + regs[DART_TTBR(&d, sid, 0) / 4] = 0x12340001 + 4 * sid; + } + assert(apple_dart_sleep_snapshot(&s, false) == 0); + assert(apple_dart_sleep_snapshot(&s, true) == 0); + assert(logs == 0); + /* Each retained value must stay exact, including the table-depth bit. */ + unsigned int changed[] = { + DART_TCR(&d, 1), DART_TCR(&d, 2), + DART_TTBR(&d, 1, 0), DART_TTBR(&d, 2, 0), + DART_TCR(&d, 0), DART_TTBR(&d, 0, 0), + hw.enable_streams, DART_T8110_PROTECT, + DART_T8110_PROTECT_LOCK, DART_T8110_V2_ERROR_MASK + }; + irqoff = true; + for (unsigned int i = 0; i < ARRAY_SIZE(changed); i++) { + regs[changed[i] / 4] ^= 8; + assert(apple_dart_sleep_snapshot(&s, true) == -EUCLEAN); + regs[changed[i] / 4] ^= 8; + assert(logs == 0); + } + irqoff = false; + d.four_level = false; + assert(apple_dart_sleep_snapshot(&s, false) == -EUCLEAN); + for (unsigned int sid = 1; sid <= 2; sid++) + regs[DART_TCR(&d, sid) / 4] = 1; + assert(apple_dart_sleep_snapshot(&s, false) == 0); + d.four_level = true; + assert(apple_dart_sleep_snapshot(&s, false) == 0); + /* Disabled, bypass, remap, unknown bits, and missing roots still fail. */ + unsigned int invalid[] = { + 0, 8, 2, 3, 5, 7, 0xb, 0xd, 0x81, 0x101, 0x11 + }; + + for (unsigned int sid = 1; sid <= 2; sid++) { + for (unsigned int i = 0; i < ARRAY_SIZE(invalid); i++) { + regs[DART_TCR(&d, sid) / 4] = invalid[i]; + assert(apple_dart_sleep_snapshot(&s, false) == + -EUCLEAN); + } + regs[DART_TCR(&d, sid) / 4] = 9; + regs[DART_TTBR(&d, sid, 0) / 4] &= ~DART_T8110_TTBR_VALID; + assert(apple_dart_sleep_snapshot(&s, false) == -EUCLEAN); + regs[DART_TTBR(&d, sid, 0) / 4] |= DART_T8110_TTBR_VALID; + } + assert(apple_dart_sleep_snapshot(&s, false) == 0); + for (unsigned int version = 0x0201; version <= 0x0203; version++) { + unsigned int mask = version >= 0x0202 ? + DART_T8110_V2_ERROR_MASK : + DART_T8110_ERROR_MASK; + unsigned int other = version >= 0x0202 ? + DART_T8110_ERROR_MASK : + DART_T8110_V2_ERROR_MASK; + d.version = version; + regs[mask / 4] = 1; + assert(apple_dart_sleep_snapshot(&s, false) == -EUCLEAN); + regs[mask / 4] = 0; + regs[other / 4] = ~0U; + assert(apple_dart_sleep_snapshot(&s, false) == 0); + regs[other / 4] = 0; + } + regs[DART_T8110_PROTECT / 4] = DART_T8110_PROTECT_TTBR_TCR; + assert(apple_dart_sleep_snapshot(&s, false) == -EUCLEAN); + regs[DART_T8110_PROTECT / 4] = 0; + fault = true; + assert(apple_dart_sleep_snapshot(&s, false) == -EBUSY); + fault = false; + regs[DART_T8110_TLB_CMD / 4] = DART_T8110_TLB_CMD_BUSY; + assert(apple_dart_sleep_snapshot(&s, false) == -EBUSY); + regs[DART_T8110_TLB_CMD / 4] = 0; + d.managed_failed = true; + assert(apple_dart_sleep_snapshot(&s, false) == -EUCLEAN); + d.managed_failed = false; + assert(apple_dart_sleep_snapshot(&s, false) == 0); + assert(apple_dart_sleep_snapshot(&s, true) == 0); + puts("PASS: DART table depths, corrupt roots, faults and snapshot validation"); +} diff --git a/tools/asahi/dart-sleep-snapshot-tests.py b/tools/asahi/dart-sleep-snapshot-tests.py new file mode 100644 index 0000000000000..5887b0b2046af --- /dev/null +++ b/tools/asahi/dart-sleep-snapshot-tests.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise retained DART snapshots with both supported translation depths.""" + +from pathlib import Path +import sleep_test_utils as x +import re +import resource +import subprocess +import tempfile + +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/iommu/apple-dart-sleep.c").read_text() +driver = (root / "drivers/iommu/apple-dart.c").read_text() +constants = ( + "DART_T8110_TLB_CMD", + "DART_T8110_TLB_CMD_BUSY", + "DART_T8110_PROTECT", + "DART_T8110_PROTECT_LOCK", + "DART_T8110_PROTECT_TTBR_TCR", + "DART_T8110_ERROR_MASK", + "DART_T8110_V2_ERROR_MASK", + "DART_T8110_ENABLE_STREAMS", + "DART_T8110_TCR", + "DART_T8110_TCR_TRANSLATE_ENABLE", + "DART_T8110_TCR_FOUR_LEVEL", + "DART_T8110_TTBR", + "DART_T8110_TTBR_VALID", +) +defines = "\n".join( + re.search(r"^#define\s+" + name + r"\s+[^\n]+", driver, re.M).group() + for name in constants +) +for name in ("DART_TCR", "DART_TTBR"): + defines += ( + "\n" + + re.search( + r"^#define " + name + r"\([^\n]+(?:\n[^\n]+)*?(?=\n\n)", driver, re.M + ).group() + ) +state = re.search(r"struct apple_dart_sleep_state \{.*?\n\};", source, re.S).group() +actual = "\n".join( + x.function(source, name) + for name in ("apple_dart_sleep_snapshot_error", "apple_dart_sleep_snapshot") +) +fixture, main = ( + Path(__file__).with_suffix(".c").read_text().split("/* ACTUAL_PRODUCTION */") +) +with tempfile.TemporaryDirectory(prefix="dart-snapshot-") as tmp: + src, binary = Path(tmp) / "test.c", Path(tmp) / "test" + for negative in (False, True): + code = actual + if negative: + old = "value &= ~dart->hw->tcr_4level;" + assert old in code + code = code.replace(old, "(void)value;") + src.write_text(fixture + defines + "\n" + state + "\n" + code + main) + subprocess.run( + [ + "cc", + "-std=gnu11", + "-Wall", + "-Wextra", + "-Werror", + str(src), + "-o", + str(binary), + ], + check=True, + ) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if negative: + assert result.returncode != 0, "Three-level-only admission was accepted" + print("PASS: old admission rejected by captured four-level regression") + else: + assert result.returncode == 0, result.stderr + print(result.stdout.strip()) diff --git a/tools/asahi/dart-sleep-writer-tests.c b/tools/asahi/dart-sleep-writer-tests.c new file mode 100644 index 0000000000000..8594dfb14dcb6 --- /dev/null +++ b/tools/asahi/dart-sleep-writer-tests.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define spin_lock_irqsave(lock, flags) \ + do { \ + (void)(lock); \ + (flags) = 0; \ + } while (0) +#define spin_unlock_irqrestore(lock, flags) \ + do { \ + (void)(lock); \ + (void)(flags); \ + } while (0) +struct apple_dart_sleep { + int result; +}; +struct apple_dart { + bool managed_sids, sleep_closed; + unsigned int sleep_writers; + int managed_lock; + struct apple_dart_sleep *sleep_record; +}; +/* ACTUAL_FUNCTIONS */ +int main(void) +{ + struct apple_dart_sleep s = { 0 }; + struct apple_dart d = { .managed_sids = true, .sleep_record = &s }; + + assert(!apple_dart_sleep_writer_get(&d)); + assert(!apple_dart_sleep_writer_get( + &d)); /* nested invalidation cannot deadlock */ + assert(apple_dart_sleep_close(&d) == -EBUSY && !d.sleep_closed); + apple_dart_sleep_writer_put(&d); + assert(apple_dart_sleep_close(&d) == -EBUSY); + apple_dart_sleep_writer_put(&d); + assert(!apple_dart_sleep_close(&d)); + assert(apple_dart_sleep_writer_get(&d) == -EBUSY && + s.result == -EUCLEAN && !d.sleep_writers); + d.sleep_writers = UINT_MAX; + d.sleep_closed = false; + assert(apple_dart_sleep_writer_get(&d) == -EOVERFLOW); + d = (struct apple_dart){ 0 }; + assert(!apple_dart_sleep_writer_get(&d)); + apple_dart_sleep_writer_put(&d); + assert(!d.sleep_writers); + puts("DART writer gate: nested/in-flight/closed/legacy invariants PASS"); +} diff --git a/tools/asahi/dart-sleep-writer-tests.py b/tools/asahi/dart-sleep-writer-tests.py new file mode 100644 index 0000000000000..ed01d2947441b --- /dev/null +++ b/tools/asahi/dart-sleep-writer-tests.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +from pathlib import Path +import subprocess, tempfile, re + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/iommu/apple-dart-sleep.c").read_text() +actual = [] +for name in ( + "apple_dart_sleep_writer_get", + "apple_dart_sleep_writer_put", + "apple_dart_sleep_close", +): + m = re.search( + r"^static (?:int|void) " + name + r"\([^;]+?\{.*?\n\}", source, re.M | re.S + ) + assert m, name + actual.append(m.group()) +f = ( + Path(__file__) + .with_suffix(".c") + .read_text() + .replace("/* ACTUAL_FUNCTIONS */", "\n".join(actual)) +) +with tempfile.TemporaryDirectory() as t: + p = Path(t) / "test.c" + p.write_text(f) + b = Path(t) / "test" + subprocess.run( + ["cc", "-std=gnu11", "-Wall", "-Wextra", "-Werror", str(p), "-o", str(b)], + check=True, + ) + subprocess.run([str(b)], check=True) diff --git a/tools/asahi/sleep_test_utils.py b/tools/asahi/sleep_test_utils.py new file mode 100644 index 0000000000000..246aeda3dfe08 --- /dev/null +++ b/tools/asahi/sleep_test_utils.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0-only +"""Extract production C functions and structures for sleep fault-injection tests.""" + +import re + + +def function(source, name): + # Keep offsets while hiding documentation and commented-out declarations. + code = re.sub( + r"/\*.*?\*/|//[^\n]*", + lambda match: re.sub(r"[^\n]", " ", match[0]), + source, + flags=re.S, + ) + match = re.search( + r"(?m)^[\w *\t]+\b" + re.escape(name) + r"\s*\([^;{}]*\)\s*\{", code + ) + assert match, f"missing exact function: {name}" + start = match.start() + opening = match.end() - 1 + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + "\n" + + +def structure(source, name): + start = source.index("struct " + name + " {") + end = source.index("\n};", start) + 3 + return source[start:end] + "\n"