diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 0cc59590ce0ed..2cf828739bc16 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -123,6 +123,26 @@ properties: may select a firmware-visible alias of the translation aperture. A consumer may further specify its own aperture in its binding. + apple,tlbi-aperture: + $ref: /schemas/types.yaml#/definitions/uint64 + description: > + T8152 Gen3 DARTs only. Physical address of this DART's 16-byte command + slot in the SoC's shared TLB invalidation aperture: the aperture base + plus the DART's offset into it. Gen3 DARTs ignore the T8110 TLB command + register; with this property, TLB invalidations are issued as a + command to the slot followed by a broadcast TLBI. + + apple,gen3-ias: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 32 + maximum: 42 + description: > + T8152 Gen3 DARTs only. Width in bits of the DVA that the stream's + table walk uses, where the loader configured a narrower walk than the + VA width the DART reports (for example 36 for a three-level walk). + Device-visible addresses above that width select an alias of the + translated window. + apple,dapf-ranges: $ref: /schemas/types.yaml#/definitions/uint64-matrix items: @@ -318,8 +338,19 @@ required: dependencies: apple,gapf-slice-values: [ 'apple,gapf-slice' ] + apple,gen3-ias: [ 'apple,tlbi-aperture' ] allOf: + - if: + properties: + compatible: + not: + contains: + const: apple,t8152-dart + then: + properties: + apple,tlbi-aperture: false + apple,gen3-ias: false - if: properties: compatible: @@ -374,6 +405,21 @@ examples: iommus = <&dart2a 0>, <&dart2b 1>; }; + - |+ + soc { + #address-cells = <2>; + #size-cells = <2>; + + dart4: iommu@3a1020000 { + compatible = "apple,t8152-dart", "apple,t8110-dart"; + reg = <0x3 0xa1020000 0x0 0x4000>; + interrupts = <1 400 4>; + #iommu-cells = <1>; + apple,tlbi-aperture = /bits/ 64 <0x200000800>; + apple,gen3-ias = <36>; + }; + }; + - |+ dart3: iommu@8e8c0000 { compatible = "apple,t8132-dart", "apple,t8110-dart"; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index b97b0c3378e45..345248957096d 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -59,6 +59,11 @@ #include "dma-iommu.h" +#ifdef CONFIG_ARM64 +#include +#include +#endif + #if IS_ENABLED(CONFIG_APPLE_DART_KUNIT_TEST) #include #endif @@ -159,6 +164,17 @@ #define DART_T8110_FLUSH_STALL_POLL 10 #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0) +/* + * T8152 Gen3 DARTs ignore TLB_CMD. A stream is invalidated through the SoC's + * shared TLBI aperture instead: a 16-byte command naming the DART's TLBI route + * and the stream, then a broadcast TLBI that the DART observes. + */ +#define DART_GEN3_TLBI_ROUTE_REG 0x4604 +#define DART_GEN3_TLBI_ROUTE_MASK GENMASK(3, 0) +#define DART_GEN3_TLBI_OP_SID BIT_ULL(60) +#define DART_GEN3_TLBI_ROUTE GENMASK_ULL(59, 56) +#define DART_GEN3_TLBI_SID GENMASK_ULL(55, 48) + #define DART_T8110_ERROR 0x100 #define DART_T8110_ERROR_STREAM GENMASK(27, 20) #define DART_T8110_ERROR_CODE GENMASK(14, 0) @@ -363,6 +379,9 @@ struct apple_dart { void __iomem *gapf; /* The GAPF slice as it read at probe; a deep sleep resets it to 0 */ u32 gapf_boot; + /* T8152 Gen3: the TLBI aperture command slot and this DART's route */ + void __iomem *gen3_tlbi; + u8 gen3_route; bool sleep_flushed; /* The firmware 27 display sleep owns it: never replay locked trees. */ struct device *m5_sleep_owner; @@ -1799,6 +1818,63 @@ static int apple_dart_t8110_tlb_wait(struct apple_dart *dart, unsigned long *fla DART_T8110_TLB_CMD); } +/* The aperture is shared by every Gen3 DART: one command at a time. */ +static DEFINE_RAW_SPINLOCK(apple_dart_gen3_tlbi_lock); + +/* + * Issue one Gen3 command: a single 16-byte store of the command and a zero + * high word, then the broadcast TLBI that consumes it. The command must reach + * the aperture before the TLBI, and the final DSB completes only when the + * broadcast has. There is no busy bit. + */ +static void apple_dart_gen3_tlbi(void __iomem *slot, u64 cmd) +{ +#ifdef CONFIG_ARM64 + asm volatile("stp %0, %1, [%2]" : : "r"(cmd), "r"(0UL), "r"(slot) : "memory"); + dsb(ish); + __tlbi(vale1os, 0); + dsb(osh); +#else + WARN_ON_ONCE(1); +#endif +} + +/* + * Invalidate every stream of @stream_map on a Gen3 DART. Like the T8110 + * command path, this is refused with -EHOSTDOWN while the DART's commands are + * gated for sleep. + */ +static int apple_dart_gen3_invalidate_tlb(struct apple_dart_stream_map *stream_map) +{ + struct apple_dart *dart = stream_map->dart; + unsigned long flags; + unsigned int sid; + + spin_lock_irqsave(&dart->lock, flags); + if (dart->commands_gated) { + spin_unlock_irqrestore(&dart->lock, flags); + return -EHOSTDOWN; + } + raw_spin_lock(&apple_dart_gen3_tlbi_lock); + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + u64 cmd = DART_GEN3_TLBI_OP_SID | + FIELD_PREP(DART_GEN3_TLBI_ROUTE, dart->gen3_route) | + FIELD_PREP(DART_GEN3_TLBI_SID, sid); + + /* The DART must see the updated tables first. */ + dma_wmb(); + apple_dart_gen3_tlbi(dart->gen3_tlbi, cmd); + } + raw_spin_unlock(&apple_dart_gen3_tlbi_lock); + spin_unlock_irqrestore(&dart->lock, flags); + return 0; +} + +static void apple_dart_gen3_unmap(void *addr) +{ + iounmap(addr); +} + static int apple_dart_t8110_hw_tlb_command_range(struct apple_dart_stream_map *stream_map, u32 command, bool range, u64 first, u64 last) @@ -2068,6 +2144,20 @@ apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID, false, 0, 0); } +/* + * A T8152 DART with a TLBI aperture is a Gen3 DART, which ignores TLB_CMD: + * every whole-stream invalidation goes through the aperture. Without the + * aperture this is the T8110 path. The ranged paths for firmware-held and + * locked DARTs still use TLB_CMD; no Gen3 DART takes them yet. + */ +static int +apple_dart_t8152_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) +{ + if (stream_map->dart->gen3_tlbi) + return apple_dart_gen3_invalidate_tlb(stream_map); + return apple_dart_t8110_hw_invalidate_tlb(stream_map); +} + static const char *apple_dart_t8110_v2_fault_name(u32 exception); /* @@ -5533,6 +5623,32 @@ static int apple_dart_probe(struct platform_device *pdev) dev_err(dev, "MMIO region too small (%pr)\n", res); return -EINVAL; } + if (of_device_is_compatible(dev->of_node, "apple,t8152-dart")) { + u64 aperture; + + ret = of_property_read_u64(dev->of_node, "apple,tlbi-aperture", &aperture); + if (!ret) { + if (!IS_ALIGNED(aperture, 16)) + return dev_err_probe(dev, -EINVAL, + "TLBI aperture %#llx not 16-byte aligned\n", + aperture); + /* Non-posted, like the DART registers. */ + dart->gen3_tlbi = ioremap_np(aperture, 16); + if (!dart->gen3_tlbi) + return -ENOMEM; + ret = devm_add_action_or_reset(dev, apple_dart_gen3_unmap, + dart->gen3_tlbi); + if (ret) + return ret; + dart->gen3_route = FIELD_GET(DART_GEN3_TLBI_ROUTE_MASK, + readl(dart->regs + DART_GEN3_TLBI_ROUTE_REG)); + dev_dbg(dev, "Gen3 TLBI aperture %#llx, route %u\n", + aperture, dart->gen3_route); + } else if (ret != -EINVAL) { + return dev_err_probe(dev, ret, "invalid apple,tlbi-aperture\n"); + } + ret = 0; + } if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "gapf")) { dart->gapf = devm_platform_ioremap_resource_byname(pdev, "gapf"); if (IS_ERR(dart->gapf)) @@ -5701,6 +5817,23 @@ static int apple_dart_probe(struct platform_device *pdev) goto err_clk_disable; } + if (dart->gen3_tlbi) { + u32 gen3_ias; + + /* + * A Gen3 stream may walk fewer address bits than PARAMS3 + * reports; DVAs are masked to the walk width before the tables + * are touched, so a device can keep its high DVA window. + */ + if (!of_property_read_u32(dev->of_node, "apple,gen3-ias", &gen3_ias) && + gen3_ias >= 32 && gen3_ias <= dart->ias) { + dev_dbg(dev, "Gen3 walk width %u (PARAMS3 VA width %u)\n", + gen3_ias, dart->ias); + dart->ias = gen3_ias; + dart->four_level = gen3_ias > 36; + } + } + dart->dma_min = 0; dart->dma_max = DMA_BIT_MASK(dart->ias); dart->dma_offset = 0; @@ -6352,6 +6485,34 @@ static const struct apple_dart_hw apple_dart_hw_t8110 = { .ttbr_count = 1, }; +/* T8110 with the Gen3 invalidation of T8152 DARTs that have a TLBI aperture. */ +static const struct apple_dart_hw apple_dart_hw_t8152 = { + .type = DART_T8110, + .irq_handler = apple_dart_t8110_irq, + .invalidate_tlb = apple_dart_t8152_hw_invalidate_tlb, + .enable_remap = apple_dart_t8110_hw_enable_remap, + .fmt = APPLE_DART2, + .max_sid_count = 256, + + .enable_streams = DART_T8110_ENABLE_STREAMS, + .lock = DART_T8110_PROTECT, + .lock_bit = DART_T8110_PROTECT_TTBR_TCR, + + .error = DART_T8110_ERROR, + + .tcr = DART_T8110_TCR, + .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE, + .tcr_disabled = 0, + .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART, + .tcr_4level = DART_T8110_TCR_FOUR_LEVEL, + + .ttbr = DART_T8110_TTBR, + .ttbr_valid = DART_T8110_TTBR_VALID, + .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT, + .ttbr_shift = DART_T8110_TTBR_SHIFT, + .ttbr_count = 1, +}; + #include "apple-dart-cio-owner.h" #include "apple-dart-sleep.c" #include "apple-dart-cio.c" @@ -7106,6 +7267,7 @@ static const struct of_device_id apple_dart_of_match[] = { { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 }, { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 }, + { .compatible = "apple,t8152-dart", .data = &apple_dart_hw_t8152 }, { .compatible = "apple,t8132-dart", .data = &apple_dart_hw_t8110 }, { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 }, {},