diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 3db18e058dce3..b45a737748180 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -274,6 +274,14 @@ properties: $ref: /schemas/types.yaml#/definitions/uint32 description: Height in pixels reserved for the display notch. + apple,pmc-interface: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The PMC interface enables of this coprocessor's display pipe, an + apple,consumer-switched interface node. The coprocessor clears them + before it powers its display domains down while the display is off, + and restores them after it powers them up again. + apple,pmp-report: $ref: /schemas/types.yaml#/definitions/phandle description: diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 9ba41ab9806d2..73a3e1764fcd8 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -132,6 +132,19 @@ properties: handoff. Linux retains occupied root slots and excludes their entire IOVA spans from allocation. This does not assert immutable descendants. + apple,clock-protection: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: PS_PROT (clock protection) register bank + - description: Index of this DART's slice in the bank + description: > + The clock protection slice that guards a power state register of the + power domain of this DART. Its consumer may power that domain down: the + DART's streams are then disabled and flushed, and the slice disabled, + before the domain powers off, and the slice is enabled again when it + powers up. Only the J713 internal display DARTs use it. + apple,tunable: $ref: /schemas/types.yaml#/definitions/uint32-matrix items: diff --git a/Documentation/devicetree/bindings/mfd/syscon.yaml b/Documentation/devicetree/bindings/mfd/syscon.yaml index 945a168b613db..4c6bfa4986ea0 100644 --- a/Documentation/devicetree/bindings/mfd/syscon.yaml +++ b/Documentation/devicetree/bindings/mfd/syscon.yaml @@ -46,6 +46,7 @@ properties: - apm,xgene-mcb - apm,xgene-rb - apm,xgene-scu + - apple,t8132-ps-prot - atmel,sama5d2-sfrbu - atmel,sama5d3-nfc-io - atmel,sama5d3-sfrbu diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 85b54bc769bfa..24f77b80d6495 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -214,12 +214,28 @@ patternProperties: maximum: 31 description: Bit of the pipe in the boost interface enable word. + apple,consumer-switched: + type: boolean + description: | + Not a power domain. The pipe's power is firmware-owned, and the + display coprocessor that refers to this node (apple,pmc-interface) + clears the bits before it powers the pipe down and gives them back + the values they had after it powers the pipe up again. + required: - - '#power-domain-cells' - - power-domains - apple,bwr-bit - apple,boost-bit + oneOf: + - required: + - '#power-domain-cells' + - power-domains + - required: + - apple,consumer-switched + properties: + '#power-domain-cells': false + power-domains: false + '^performance-controller@[0-9a-f]+$': type: object description: The host performance controller's voter agent. diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index d9cf0b9129926..8281b883bd2ac 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -28,6 +28,19 @@ allOf: apple,force-disable: false apple,force-reset: false apple,externally-clocked: false + apple,consumer-switched: false + - if: + required: + - apple,consumer-switched + then: + required: + - apple,firmware-owned + - if: + required: + - apple,retention-workaround + then: + required: + - apple,consumer-switched description: | Apple SoCs include PMGR blocks responsible for power management, @@ -132,6 +145,32 @@ properties: the power-state register. type: boolean + apple,consumer-switched: + description: + With apple,firmware-owned, the consumer of this domain may power it + off and on again around stopping its coprocessor, through the PMGR + driver, which then writes the state register. The OS otherwise treats + the domain as firmware-owned and always on. A chain of such domains + (this one and its consumer-switched parents) is powered off child first + and on parent first. + type: boolean + + apple,retention-workaround: + description: + PMGR registers to set while this consumer-switched domain is off, and + to restore afterwards, as triplets of the offset in the PMGR register + block, a mask and the masked value while the domain is off. They are + set after automatic power management of the domain is disabled and + before it powers off, and restored after it powers on and before + automatic power management is enabled again. + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + items: + - description: register offset + - description: mask + - description: value while the domain is off + maxItems: 4 + apple,inherited-on: description: | On T8132, the boot loader hands over an active domain whose state diff --git a/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi index 7f0e91c4002eb..e6a1ab6ebeb34 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-j713-pmgr.dtsi @@ -105,13 +105,16 @@ status = "disabled"; }; +/* The DCP powers these down while the display is off. */ &ps_disp_sys { apple,firmware-owned; + apple,consumer-switched; status = "okay"; }; &ps_disp_fe { apple,firmware-owned; + apple,consumer-switched; status = "okay"; }; @@ -531,6 +534,9 @@ &ps_disp_cpu { apple,firmware-owned; + apple,consumer-switched; + /* Power-gate retention workaround, as macOS sets it while off. */ + apple,retention-workaround = <0x2504c 0x3 0x3>, <0x25070 0x1 0x0>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8132-j713.dts b/arch/arm64/boot/dts/apple/t8132-j713.dts index b4a0813268ebf..94bfc49c5e279 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713.dts +++ b/arch/arm64/boot/dts/apple/t8132-j713.dts @@ -150,3 +150,19 @@ firmware-name = "apple/mesacal-j713.bin", "apple/msrk-j713.bin"; }; +/* + * The DCP powers its display domains down while blanked + * (appledrm.power_gate_ms): the clock protection slices of its DARTs, as the + * ADT has them, and the internal pipe's PMC interface. + */ +&dcp_dart { + apple,clock-protection = <&ps_prot 42>; +}; + +&disp0_dart { + apple,clock-protection = <&ps_prot 8>; +}; + +&dcp { + apple,pmc-interface = <&pmc_disp0>; +}; diff --git a/arch/arm64/boot/dts/apple/t8132-memory.dtsi b/arch/arm64/boot/dts/apple/t8132-memory.dtsi index c6cafa8219d2a..e0de465b9ffe1 100644 --- a/arch/arm64/boot/dts/apple/t8132-memory.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-memory.dtsi @@ -24,6 +24,18 @@ * boost interface enables. The PMC ignores a pipe's bandwidth * and display clock votes while they are clear. */ + /* + * The internal display pipe's enables. Its power states are + * firmware-owned, so they are not a power domain: its DCP + * clears them before powering the pipe down and restores them. + */ + pmc_disp0: interface-disp0 { + label = "disp0-interface"; + apple,consumer-switched; + apple,bwr-bit = <0>; + apple,boost-bit = <0>; + }; + pmc_dispext0: interface-dispext0 { label = "dispext0-interface"; #power-domain-cells = <0>; diff --git a/arch/arm64/boot/dts/apple/t8132.dtsi b/arch/arm64/boot/dts/apple/t8132.dtsi index 35490237bba94..d6a01e1473200 100644 --- a/arch/arm64/boot/dts/apple/t8132.dtsi +++ b/arch/arm64/boot/dts/apple/t8132.dtsi @@ -474,10 +474,17 @@ compatible = "apple,t8132-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; #size-cells = <1>; - reg = <0x3 0x80700000 0 0x14000>; + /* Up to the display CPU's power-gate retention registers. */ + reg = <0x3 0x80700000 0 0x26000>; /* child nodes are added in t8132-pmgr.dtsi */ }; + /* Clock protection slices guarding power states of DART domains. */ + ps_prot: syscon@38079c000 { + compatible = "apple,t8132-ps-prot", "syscon"; + reg = <0x3 0x8079c000 0 0x4000>; + }; + aic: interrupt-controller@381000000 { compatible = "apple,t8132-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f233c079997cc..d098c43dbe72b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -6683,7 +6683,9 @@ static int dcp_platform_suspend(struct device *dev) /* * T8132: setPowerState powers the display down; the coprocessor keeps - * running and its power domains stay on, as below. + * running and its power domains stay on, as below, unless the internal + * display is blanked with appledrm.power_gate_ms set: then the + * coprocessor and its domains are powered down too (iomfb_m4_suspend()). */ if (iomfb_is_m4(dcp)) { int ret = iomfb_m4_suspend(dcp); diff --git a/drivers/gpu/drm/apple/iomfb_m4.c b/drivers/gpu/drm/apple/iomfb_m4.c index 48b7179c439e4..b6b301a20ab20 100644 --- a/drivers/gpu/drm/apple/iomfb_m4.c +++ b/drivers/gpu/drm/apple/iomfb_m4.c @@ -24,11 +24,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -39,6 +41,8 @@ #include #include #include +#include +#include #include "afk.h" #include "dcp.h" @@ -82,6 +86,32 @@ module_param_cb(idle_cache_ms, &idle_cache_ms_ops, &idle_cache_ms, 0644); MODULE_PARM_DESC(idle_cache_ms, "J713: cache a static frame after this many ms without a flip (0: never, at most 60000; applies from the next swap)"); +static unsigned int power_gate_ms; + +static int power_gate_ms_set(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, 0, 600000); +} + +static const struct kernel_param_ops power_gate_ms_ops = { + .set = power_gate_ms_set, + .get = param_get_uint, +}; + +module_param_cb(power_gate_ms, &power_gate_ms_ops, &power_gate_ms, 0644); +MODULE_PARM_DESC(power_gate_ms, + "J713: power the display coprocessor and its domains down after this many ms blanked, and before system sleep (0: never, at most 600000)"); + +/* + * DCP CPU control and status, in the coprocessor's ASC block. The idle bit + * is dcp.c's APPLE_DCP_COPROC_CPU_STATUS_IDLE; macOS read status 0x6e at + * its idle check before clearing RUN. + */ +#define DCP_M4_CPU_CONTROL 0x44 +#define DCP_M4_CPU_RUN BIT(4) +#define DCP_M4_CPU_STATUS 0x48 +#define DCP_M4_CPU_STATUS_IDLE BIT(5) + /* One timing of an external sink for its DRM connector. */ struct iomfb_m4_ext_mode { struct drm_display_mode mode; @@ -153,6 +183,17 @@ struct iomfb_m4 { bool display_off; /* setPowerState(0) while blanked, until the next frame */ u64 display_off_cycles; s64 display_off_us, display_off_us_max, display_on_us, display_on_us_max; + /* + * Internal J713 DCP that can power its CPU and display domains down + * while blanked (apple,pmc-interface; power_gate_ms), and whether they + * are down: then no register of the DCP, its mailbox or its display + * DARTs is used until iomfb_m4_power_ungate(). + */ + bool gate_capable; + bool gated; + struct delayed_work gate_work; + u64 gate_cycles; + s64 gate_off_us, gate_on_us; /* Exact completed successor or blank is the only framebuffer retirement. */ struct drm_framebuffer *active_fb; struct drm_framebuffer *uncertain_fb; @@ -216,6 +257,9 @@ static void dcp_m4_runtime_wake(struct iomfb_m4 *dcp); struct dcp_m4_runtime; static void dcp_m4_ext_async_work(struct work_struct *work); static int dcp_m4_set_power_state(struct iomfb_m4 *dcp, bool on); +static int iomfb_m4_power_gate(struct iomfb_m4 *dcp); +static int iomfb_m4_power_ungate(struct iomfb_m4 *dcp); +static void iomfb_m4_gate_work(struct work_struct *work); static void dcp_m4_runtime_receive(struct iomfb_m4 *dcp, u64 word); static int dcp_m4_dart_check(struct device *dev) @@ -1025,6 +1069,10 @@ static int dcp_m4_runtime_health(struct dcp_m4_runtime *r) struct iomfb_m4 *dcp = r->dcp; int ret = READ_ONCE(dcp->error); + /* Powered down by its owner: only the failure latch. */ + if (READ_ONCE(dcp->gated)) + return ret; + if (!ret && (!(readl(dcp->coproc + 0x44) & BIT(4)) || dcp_m4_dart_check(dcp->dev) || (!dcp->external && dcp_m4_dart_check(&dcp->piodma->dev)))) ret = -EIO; @@ -1977,6 +2025,14 @@ static void dcp_m4_runtime_idle(struct work_struct *work) int ret; mutex_lock(&dcp->lock); + /* + * Powered down (iomfb_m4_power_gate()): nothing arrives then, and + * nothing may reach the stopped firmware or its powered-down DARTs. + */ + if (READ_ONCE(dcp->gated)) { + mutex_unlock(&dcp->lock); + return; + } ret = dcp_m4_runtime_drain(r, 32); if (ret < 0) { dcp_m4_transaction_fail(&dcp->transaction, ret); @@ -2587,6 +2643,7 @@ int iomfb_m4_init(struct apple_dcp *owner) INIT_WORK(&dcp->ext_put_work, iomfb_m4_ext_put_work); init_waitqueue_head(&dcp->ext_settle_wq); INIT_DELAYED_WORK(&dcp->idle_cache, iomfb_m4_idle_cache_work); + INIT_DELAYED_WORK(&dcp->gate_work, iomfb_m4_gate_work); if (!request_mem_region(res->start, resource_size(res), dev_name(dev))) { ret = -EBUSY; goto release; @@ -2613,6 +2670,9 @@ int iomfb_m4_init(struct apple_dcp *owner) if (ret) return ret; dcp->external = external; + /* The internal display's power-down: its PMC interface names the rest. */ + dcp->gate_capable = !external && apple_pmp_j713_board() && + of_property_present(dev->of_node, "apple,pmc-interface"); owner->v26_6 = dcp; owner->rtk = dcp->rtkit; owner->coproc_reg = dcp->coproc; @@ -3254,12 +3314,16 @@ int iomfb_m4_present(struct apple_dcp *owner, struct drm_framebuffer *fb, fb->pitches[0], fb->format->format == DRM_FORMAT_ARGB8888); if (ret) return ret; + /* A power-down not yet started is not needed; one under way finishes first. */ + cancel_delayed_work(&dcp->gate_work); mutex_lock(&dcp->lock); ret = dcp_m4_runtime_live(dcp->runtime); if (!ret && dcp->powered_off) ret = -EAGAIN; if (!ret) ret = iomfb_m4_check_frame(&frame); + if (!ret && dcp->gated) + ret = iomfb_m4_power_ungate(dcp); /* The first frame after a blank powers the display back up. */ if (!ret && dcp->display_off) { ktime_t start = ktime_get(); @@ -3332,6 +3396,11 @@ int iomfb_m4_blank(struct apple_dcp *owner) dcp->display_off_us = ktime_us_delta(ktime_get(), start); dcp->display_off_us_max = max(dcp->display_off_us, dcp->display_off_us_max); + unsigned int gate_ms = READ_ONCE(power_gate_ms); + + if (dcp->gate_capable && gate_ms) + mod_delayed_work(system_dfl_wq, &dcp->gate_work, + msecs_to_jiffies(gate_ms)); } } } @@ -3381,6 +3450,222 @@ static int dcp_m4_set_power_state(struct iomfb_m4 *dcp, bool on) return ret; } +/* Stop or start the DCP CPU; stopping waits for its idle status first. */ +static int iomfb_m4_cpu_run(struct iomfb_m4 *dcp, bool run) +{ + u32 control = readl(dcp->coproc + DCP_M4_CPU_CONTROL), status; + int ret; + + if (!!(control & DCP_M4_CPU_RUN) == run) + return -EBUSY; + if (!run) { + ret = readl_poll_timeout(dcp->coproc + DCP_M4_CPU_STATUS, status, + status & DCP_M4_CPU_STATUS_IDLE, 100, 100000); + if (ret) + return ret; + } + control = run ? control | DCP_M4_CPU_RUN : control & ~DCP_M4_CPU_RUN; + writel(control, dcp->coproc + DCP_M4_CPU_CONTROL); + return readl(dcp->coproc + DCP_M4_CPU_CONTROL) == control ? 0 : -EIO; +} + +static int iomfb_m4_cpu_start(void *data) +{ + return iomfb_m4_cpu_run(data, true); +} + +/* + * Whether every provider of the power-down can take part now, changing + * nothing: its display DARTs, the PMP vote (held, and releasable without + * losing it), the PMC interface enables and the PMGR states. + */ +static int iomfb_m4_gate_check(struct iomfb_m4 *dcp) +{ + return apple_dart_dcp_power_check(dcp->dev) ?: + apple_dart_dcp_power_check(&dcp->piodma->dev) ?: + apple_pmp_j713_display_check_release(dcp->dev) ?: + apple_pmc_interface_consumer_check(dcp->dev) ?: + apple_pmgr_ps_consumer_check(dcp->dev); +} + +/* + * After the blank's setPowerState(0), power the DCP and the display domains + * down as macOS does for display sleep: release the AP link (A500), let RTKit + * idle, stop the CPU, then power down its display DARTs, release its PMP + * vote, clear its PMC interface enables and power its CPU, frontend and + * system domains off. Firmware, endpoints and shared buffers stay in place + * for iomfb_m4_power_ungate(). Before the first step every provider is + * asked whether it can take part: if one cannot, nothing changes and the + * display just stays logically off. A failure after that fails the session. + */ +static int iomfb_m4_power_gate(struct iomfb_m4 *dcp) +{ + struct dcp_m4_runtime *r = dcp->runtime; + ktime_t start = ktime_get(); + unsigned long flags; + __le32 zero = 0; + const char *step; + int ret; + + lockdep_assert_held(&dcp->lock); + if (!dcp->gate_capable || dcp->gated || !dcp->display_off || dcp->powered_off || + dcp->active_fb || dcp->active_frame.dva || dcp->transaction.calls) + return -EBUSY; + ret = dcp_m4_runtime_live(r); + if (ret) + return ret; + ret = iomfb_m4_gate_check(dcp); + if (ret) { + dev_dbg(dcp->dev, "display power-down not possible now (%d): stays logically off\n", + ret); + return ret; + } + + step = "A500"; + ret = dcp_m4_runtime_call(r, 500, false, &zero, sizeof(zero), NULL, 0); + if (!ret) { + step = "drain"; + ret = dcp_m4_runtime_drain(r, 64); + if (ret > 0 || (!ret && r->count)) + ret = -EBUSY; + } + if (!ret) { + step = "RTKit idle"; + ret = apple_rtkit_idle(dcp->rtkit); + } + if (ret) + goto fail; + apple_rtkit_hold_rx(dcp->rtkit); + step = "CPU stop"; + ret = iomfb_m4_cpu_run(dcp, false); + if (ret) + goto fail; + /* + * The stopped CPU sends nothing more. A message it sent during the idle + * handshake would be handled after the power-down, by firmware that is + * stopped and through DARTs that are off: take everything in now, and + * fail here, with nothing powered down, if anything arrived. + */ + step = "late message"; + ret = apple_rtkit_poll(dcp->rtkit); + if (ret >= 0) { + apple_rtkit_flush_rx(dcp->rtkit); + spin_lock_irqsave(&dcp->state_lock, flags); + ret = r->count ? -EBUSY : 0; + spin_unlock_irqrestore(&dcp->state_lock, flags); + } + if (ret) + goto fail; + + /* From here on nothing may use the DCP's or its DARTs' registers. */ + WRITE_ONCE(dcp->gated, true); + step = "DART"; + ret = apple_dart_dcp_power_off(dcp->dev); + if (!ret) + ret = apple_dart_dcp_power_off(&dcp->piodma->dev); + if (!ret) { + step = "PMP"; + ret = apple_pmp_j713_display_put(dcp->dev); + } + if (!ret) { + step = "PMC"; + ret = apple_pmc_interface_consumer_off(dcp->dev); + } + if (!ret) { + step = "PMGR"; + ret = apple_pmgr_ps_consumer_off(dcp->dev); + } + if (ret) + goto fail; + dcp->gate_cycles++; + dcp->gate_off_us = ktime_us_delta(ktime_get(), start); + return 0; +fail: + dev_err(dcp->dev, "display power-down failed at %s: %d\n", step, ret); + dcp_m4_failed(dcp, ret); + return ret; +} + +/* + * Power the DCP back up for the first frame after a gated blank, in macOS's + * order: the domains, the PMP vote, the PMC interface enables, then the + * display DARTs, then the CPU through RTKit's retained wake, and the display + * preparation macOS sends before setPowerState(1). A failure fails the + * session. + */ +static int iomfb_m4_power_ungate(struct iomfb_m4 *dcp) +{ + struct dcp_m4_runtime *r = dcp->runtime; + ktime_t start = ktime_get(); + u8 prepare[8] = {}, prepared[8] = {}; + __le32 main_display = 0; + const char *step; + int ret; + + lockdep_assert_held(&dcp->lock); + if (!dcp->gated) + return 0; + step = "PMGR"; + ret = apple_pmgr_ps_consumer_on(dcp->dev); + if (!ret) { + step = "PMP"; + ret = apple_pmp_j713_display_get(dcp->dev); + } + if (!ret) { + step = "PMC"; + ret = apple_pmc_interface_consumer_on(dcp->dev); + } + if (!ret) { + step = "DART"; + ret = apple_dart_dcp_power_on(dcp->dev); + } + if (!ret) + ret = apple_dart_dcp_power_on(&dcp->piodma->dev); + if (!ret) { + step = "RTKit wake"; + ret = apple_rtkit_wake_retained(dcp->rtkit, iomfb_m4_cpu_start, dcp); + } + if (ret) + goto fail; + WRITE_ONCE(dcp->gated, false); + + step = "A414"; + ret = dcp_m4_runtime_call(r, 414, false, prepare, sizeof(prepare), + prepared, sizeof(prepared)); + if (!ret && get_unaligned_le32(prepared + 4)) + ret = -EREMOTEIO; + if (!ret) { + step = "A412"; + ret = dcp_m4_runtime_call(r, 412, false, NULL, 0, &main_display, + sizeof(main_display)); + } + if (!ret && le32_to_cpu(main_display) != 1) + ret = -EREMOTEIO; + if (ret) + goto fail; + dcp->gate_on_us = ktime_us_delta(ktime_get(), start); + return 0; +fail: + dev_err(dcp->dev, "display power-up failed at %s: %d\n", step, ret); + dcp_m4_failed(dcp, ret); + return ret; +} + +static void iomfb_m4_gate_work(struct work_struct *work) +{ + struct iomfb_m4 *dcp = container_of(to_delayed_work(work), struct iomfb_m4, gate_work); + + mutex_lock(&dcp->lock); + /* + * Still blanked and not yet down: a frame since then cancels it, and so + * does turning the power-down off meanwhile (power_gate_ms = 0). + */ + if (dcp->runtime && dcp->display_off && !dcp->gated && !READ_ONCE(dcp->error) && + !READ_ONCE(dcp->owner->quiescing) && READ_ONCE(power_gate_ms)) + iomfb_m4_power_gate(dcp); + mutex_unlock(&dcp->lock); +} + /* A364: the firmware's record of a timing and colour pair. */ struct dcp_m4_mode_info_req { u8 mode[0x64]; @@ -4976,14 +5261,23 @@ int iomfb_m4_suspend(struct apple_dcp *owner) } if (!dcp || !dcp->runtime) return -ENODEV; + cancel_delayed_work_sync(&dcp->gate_work); mutex_lock(&dcp->lock); /* Also the startup surface, shown before the first frame */ if (dcp->active_fb || dcp->active_frame.dva) { dev_warn(dcp->dev, "panel still scanning out: refusing system sleep\n"); ret = -EBUSY; } else if (dcp->display_off) { - /* Already powered down by the blank; the next frame powers it up. */ + /* + * Already powered down by the blank; the next frame powers it up. + * Power the DCP and its domains down too, if it may. That is an + * extra saving, not a condition of sleep: a power-down that cannot + * start leaves the display logically off, and one that fails has + * failed the session already; neither refuses the sleep. + */ ret = 0; + if (dcp->gate_capable && READ_ONCE(power_gate_ms) && !dcp->gated) + iomfb_m4_power_gate(dcp); } else { ret = dcp_m4_set_power_state(dcp, false); if (!ret) @@ -5011,7 +5305,10 @@ int iomfb_m4_resume(struct apple_dcp *owner) if (!dcp || !dcp->runtime) return -ENODEV; mutex_lock(&dcp->lock); - if (dcp->powered_off) { + if (dcp->gated) { + /* Stays down until the next frame. */ + ret = 0; + } else if (dcp->powered_off) { ret = dcp_m4_set_power_state(dcp, true); if (!ret) dcp->powered_off = false; @@ -5079,6 +5376,7 @@ void iomfb_m4_quiesce(struct apple_dcp *owner) flush_workqueue(dcp->runtime->wq); /* After the flush, as drained callbacks can queue it. */ cancel_delayed_work_sync(&dcp->idle_cache); + cancel_delayed_work_sync(&dcp->gate_work); } static int iomfb_m4_status_show(struct seq_file *s, void *unused) @@ -5097,7 +5395,11 @@ static int iomfb_m4_status_show(struct seq_file *s, void *unused) dcp_m4_failed(dcp, health); seq_printf(s, "firmware=26.6.2 ready=%u qualified=%u error=%d run=%u health=%d\n", dcp->ready, r && r->qualified, dcp->error, - !!(readl(dcp->coproc + 0x44) & BIT(4)), health); + !dcp->gated && !!(readl(dcp->coproc + DCP_M4_CPU_CONTROL) & DCP_M4_CPU_RUN), + health); + seq_printf(s, "gate_capable=%u gated=%u gate_cycles=%llu gate_off_us=%lld gate_on_us=%lld\n", + dcp->gate_capable, dcp->gated, dcp->gate_cycles, dcp->gate_off_us, + dcp->gate_on_us); seq_printf(s, "startup=%u background=%u power=%u scanout=%u callbacks=%u\n", r && r->done, r && r->background_done, r && r->power_done, r && r->scanout_done, r ? r->callbacks : 0); diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 03a3c6d3759e1..dd2dd23043b54 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -319,6 +319,7 @@ config APPLE_DART config APPLE_DART_J713_DCP bool "Retained J713 display DART support" depends on APPLE_DART=y && ARM64_16K_PAGES + select MFD_SYSCON help Adopt the qualified running J713 internal display DARTs. Their firmware-visible allocations, page tables and noncoherent CPU diff --git a/drivers/iommu/apple-dart-dcp-shared.h b/drivers/iommu/apple-dart-dcp-shared.h index f597edd6b6128..2396ac5e1d33c 100644 --- a/drivers/iommu/apple-dart-dcp-shared.h +++ b/drivers/iommu/apple-dart-dcp-shared.h @@ -378,7 +378,7 @@ int apple_dart_dcp_dispext_stream_ok(struct device *piodma, unsigned int sid) if (apple_dart_dispext_retired(pio->dart)) return -EIO; /* Its power not confirmed since sleep: usable later, not now. */ - return apple_dart_dispext_unconfirmed(pio->dart) ? -EAGAIN : 0; + return apple_dart_pm_unconfirmed(pio->dart) ? -EAGAIN : 0; } EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_stream_ok); @@ -807,7 +807,7 @@ static int __apple_dart_dcp_shared_map(struct device *dev, struct device *piodma /* * J713 external DCP: the external display DART operations above, each - * inside one use of the DART's registers (apple_dart_dispext_get()): with + * inside one use of the DART's registers (apple_dart_pm_get()): with * its power not confirmed since sleep, -EAGAIN before any step, nothing * published, nothing retired. A sleep waits for a use under way to end, so * no step of it (its flushes included) runs unconfirmed. @@ -824,12 +824,12 @@ static struct apple_dart *apple_dart_dcp_dispext_gate(struct device *piodma) struct apple_dart *__gd = apple_dart_dcp_dispext_gate(piodma); \ typeof(call) __ret; \ \ - if (__gd && !apple_dart_dispext_get(__gd)) { \ + if (__gd && !apple_dart_pm_get(__gd)) { \ __ret = (typeof(call))-EAGAIN; \ } else { \ __ret = (call); \ if (__gd) \ - apple_dart_dispext_put(__gd); \ + apple_dart_pm_put(__gd); \ } \ __ret; \ }) @@ -863,11 +863,11 @@ void *apple_dart_dcp_dispext_scanout_alloc(struct device *piodma, size_t size, u struct apple_dart *dart = apple_dart_dcp_dispext_gate(piodma); void *ret; - if (dart && !apple_dart_dispext_get(dart)) + if (dart && !apple_dart_pm_get(dart)) return ERR_PTR(-EAGAIN); ret = __apple_dart_dcp_dispext_scanout_alloc(piodma, size, dva); if (dart) - apple_dart_dispext_put(dart); + apple_dart_pm_put(dart); return ret; } EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_scanout_alloc); diff --git a/drivers/iommu/apple-dart-dcp.h b/drivers/iommu/apple-dart-dcp.h index 6e74f56c5132c..49ed48c2f3de6 100644 --- a/drivers/iommu/apple-dart-dcp.h +++ b/drivers/iommu/apple-dart-dcp.h @@ -68,65 +68,96 @@ struct apple_dart_dcp { } while (0) /* - * J713 external display DART: its power not confirmed since sleep (from - * suspend until its firmware powers it up again): no register of it may be - * read or written, and nothing is retired for that. + * A power-gated display DART (see struct apple_dart): its power not + * confirmed since it may have gone off (from sleep, or from a power-down by + * its DCP, until it is confirmed on again): no register of it may be read or + * written, and nothing is retired for that. */ -static inline bool apple_dart_dispext_unconfirmed(const struct apple_dart *dart) +static inline bool apple_dart_pm_unconfirmed(const struct apple_dart *dart) { - return dart->dispext && READ_ONCE(dart->dispext_unverified); + return dart->power_gated && READ_ONCE(dart->pm_unverified); } /* - * A use of an external display DART's registers, taken at the start of an + * A use of a power-gated display DART's registers, taken at the start of an * operation and dropped at its end, and only with its power confirmed: - * false (the caller returns -EAGAIN, nothing retired) from sleep until its - * next confirmed power-up. Inside it the operation's steps, flushes included, - * run unchecked: an invalidation (sleep) waits for it to end. Any other - * DART: always true, no count. + * false (the caller returns -EAGAIN, nothing retired) from the moment it may + * go off until its next confirmed power-up. Inside it the operation's steps, + * flushes included, run unchecked: an invalidation waits for it to end. Any + * other DART: always true, no count. */ -static inline bool apple_dart_dispext_get(struct apple_dart *dart) +static inline bool apple_dart_pm_get(struct apple_dart *dart) { unsigned long flags; bool ok; - if (!dart->dispext) + if (!dart->power_gated) return true; - spin_lock_irqsave(&dart->dispext_pm_lock, flags); - ok = !dart->dispext_unverified; + spin_lock_irqsave(&dart->pm_gate_lock, flags); + ok = !dart->pm_unverified; if (ok) - dart->dispext_users++; - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + dart->pm_users++; + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); return ok; } -static inline void apple_dart_dispext_put(struct apple_dart *dart) +static inline void apple_dart_pm_put(struct apple_dart *dart) { unsigned long flags; - if (!dart->dispext) + if (!dart->power_gated) return; - spin_lock_irqsave(&dart->dispext_pm_lock, flags); - if (!--dart->dispext_users) - wake_up_all(&dart->dispext_wq); - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + spin_lock_irqsave(&dart->pm_gate_lock, flags); + if (!--dart->pm_users) + wake_up_all(&dart->pm_wq); + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); } /* - * Sleep: no power confirmed from now on, a new generation (a check - * still under way confirms nothing), and every use in flight waited for. - * @suspend: asleep until the matching resume (no check runs then). + * Sleep or a power-down: no power confirmed from now on, a new generation (a + * check still under way confirms nothing), and every use in flight waited + * for. @suspend: asleep until the matching resume (no check runs then). */ -static inline void apple_dart_dispext_invalidate(struct apple_dart *dart, bool suspend) +static inline void apple_dart_pm_invalidate(struct apple_dart *dart, bool suspend) { unsigned long flags; - spin_lock_irqsave(&dart->dispext_pm_lock, flags); - dart->dispext_unverified = true; - dart->dispext_sleeping = suspend; - dart->dispext_gen++; - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); - wait_event(dart->dispext_wq, !READ_ONCE(dart->dispext_users)); + spin_lock_irqsave(&dart->pm_gate_lock, flags); + dart->pm_unverified = true; + dart->pm_sleeping = suspend; + dart->pm_gen++; + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); + wait_event(dart->pm_wq, !READ_ONCE(dart->pm_users)); +} + +/* + * The locks of every borrowed stream of @dart, in a fixed order (SID23, then + * SID0, then SID16). An internal display DART's host_off changes only with + * all of them held, so under any one of them it is stable: while it is set, + * the DART's DCP has it powered down (or powering down), and its generic DMA + * streams (SID0, SID23) update their page tables in memory only, with no + * register used; its power-up flushes every translation, with these locks + * held, before it enables the streams again. + */ +static void apple_dart_host_lock(struct apple_dart *dart, unsigned long *flags) +{ + struct apple_dart_dcp *streams[] = { dart->dcp, dart->frontend[0], dart->frontend[1] }; + unsigned int held = 0; + + local_irq_save(*flags); + for (unsigned int i = 0; i < ARRAY_SIZE(streams); i++) + if (streams[i]) + spin_lock_nested(&streams[i]->lock, held++); +} + +static void apple_dart_host_unlock(struct apple_dart *dart, unsigned long flags) +{ + struct apple_dart_dcp *streams[] = { dart->frontend[1], dart->frontend[0], dart->dcp }; + + for (unsigned int i = 0; i < ARRAY_SIZE(streams); i++) + if (streams[i]) + spin_unlock(&streams[i]->lock); + local_irq_restore(flags); } /* Every borrowed stream of @dart, for one reason. */ @@ -207,7 +238,7 @@ static bool apple_dart_dcp_registers(struct apple_dart_dcp *dcp) struct apple_dart *dart = dcp->dart; u32 protect, ttbr, tcr, streams; - /* An external display DART's callers hold a use of it (apple_dart_dispext_get()). */ + /* An external display DART's callers hold a use of it (apple_dart_pm_get()). */ if (READ_ONCE(dcp->failed)) return false; protect = readl(dart->regs + DART_T8110_PROTECT); @@ -639,11 +670,23 @@ static phys_addr_t apple_dart_dcp_translate(struct apple_dart_dcp *dcp, u64 iova phys = dcp_dart_pte_phys(pte) | (iova & (DCP_DART_PAGE - 1)); goto out; } + /* + * Powered down by its DCP (stable under this lock): no register, so + * only the host-owned leaves, which expected[] mirrors. + */ + if (READ_ONCE(dcp->dart->host_off)) { + if (!test_bit(root, dcp->foreign) && dcp->expected[root]) { + pte = dcp->expected[root][leaf]; + if (pte) + phys = dcp_dart_pte_phys(pte) | (iova & (DCP_DART_PAGE - 1)); + } + goto out; + } /* * An external display DART's registers only inside a use of them (its * power confirmed, a sleep waits for it): none, no lookup. */ - if (dcp->failed || !apple_dart_dispext_get(dcp->dart)) + if (dcp->failed || !apple_dart_pm_get(dcp->dart)) goto out; used = true; if (!apple_dart_dcp_registers(dcp)) @@ -675,7 +718,7 @@ static phys_addr_t apple_dart_dcp_translate(struct apple_dart_dcp *dcp, u64 iova out: spin_unlock_irqrestore(&dcp->lock, flags); if (used) - apple_dart_dispext_put(dcp->dart); + apple_dart_pm_put(dcp->dart); return phys; } @@ -1366,6 +1409,14 @@ static size_t apple_dart_dcp_unmap(struct apple_dart_dcp *dcp, u64 iova, { size_t unmapped; + /* + * An internal display DART that its DCP powers down: no use of its + * registers is needed. Under the stream's lock __apple_dart_dcp_unmap() + * sees whether the DART is powered (apple_dart_host_lock()), and while it + * is not, only updates the page tables. + */ + if (dcp->dart->ps_prot) + return __apple_dart_dcp_unmap(dcp, iova, size, count, gather); /* * On an external display DART, only inside a use of its registers. * Returning 0 (nothing unmapped) is not safe for general DMA teardown: @@ -1375,13 +1426,13 @@ static size_t apple_dart_dcp_unmap(struct apple_dart_dcp *dcp, u64 iova, * has no generic DMA user (its host mappings are the private window and * scanout APIs, which never come through here). */ - if (!apple_dart_dispext_get(dcp->dart)) { + if (!apple_dart_pm_get(dcp->dart)) { dev_err_ratelimited(dcp->dart->dev, "SID%u unmap at %llx while unconfirmed since sleep: refused\n", dcp->sid, iova); return 0; } unmapped = __apple_dart_dcp_unmap(dcp, iova, size, count, gather); - apple_dart_dispext_put(dcp->dart); + apple_dart_pm_put(dcp->dart); return unmapped; } @@ -1419,8 +1470,13 @@ static size_t __apple_dart_dcp_unmap(struct apple_dart_dcp *dcp, u64 iova, while (cleared < count && dcp->table[root][leaf + cleared]) WRITE_ONCE(dcp->table[root][leaf + cleared++], 0); dma_wmb(); - /* Strict SID0: invalidate before the IOVA or its pages can be reused. */ - if (cleared && dcp->sid == 0 && apple_dart_dcp_flush(dcp)) { + /* + * Strict SID0: invalidate before the IOVA or its pages can be reused. + * Not while its DCP has the DART powered down: nothing is cached then, + * and the power-up flushes before it enables the streams. + */ + if (cleared && dcp->sid == 0 && !READ_ONCE(dcp->dart->host_off) && + apple_dart_dcp_flush(dcp)) { /* A stale translation may survive: republish the leaves and keep the pages. */ for (size_t i = 0; i < cleared; i++) WRITE_ONCE(dcp->table[root][leaf + i], dcp->expected[root][leaf + i]); diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 5f512e18bfd50..b3c95c3067e45 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -963,6 +963,194 @@ static void apple_dart_test_strict_engines(struct kunit *test) "%s", lazy[i]); } +#if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) +#define APPLE_DART_TEST_PS_SLICE 42 + +static void apple_dart_test_regmap_exit(void *map) +{ + regmap_exit(map); +} + +/* An internal display DART with one live stream and a PS_PROT bank in RAM. */ +static struct apple_dart *apple_dart_test_host_dart(struct kunit *test) +{ + static const struct regmap_config cfg = { + .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .max_register = APPLE_PS_PROT_SLICE(APPLE_PS_PROT_SLICES - 1), + }; + static atomic_t instance = ATOMIC_INIT(0); + struct apple_dart *dart; + char name[40]; + void *bank; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + /* Device names must be unique: a test may set up several. */ + snprintf(name, sizeof(name), "apple-dart-host-power-%d", atomic_inc_return(&instance)); + dart->dev = kunit_device_register(test, name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->dev); + dev_set_drvdata(dart->dev, dart); + /* Up to the exception registers at 0x4000, which a 0x0203 DART reads. */ + dart->regs = (__force void __iomem *)kunit_kzalloc(test, APPLE_DART_TEST_REGS_SIZE, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart->regs); + dart->hw = &apple_dart_hw_t8110; + dart->num_streams = 2; + dart->pgsize = SZ_16K; + dart->version = 0x0203; + dart->power_gated = true; + spin_lock_init(&dart->lock); + spin_lock_init(&dart->pm_gate_lock); + init_waitqueue_head(&dart->pm_wq); + pm_runtime_set_active(dart->dev); + pm_runtime_enable(dart->dev); + + bank = kunit_kzalloc(test, APPLE_PS_PROT_SLICE(APPLE_PS_PROT_SLICES), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bank); + dart->ps_prot = regmap_init_mmio(NULL, (__force void __iomem *)bank, &cfg); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dart->ps_prot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, apple_dart_test_regmap_exit, + dart->ps_prot), 0); + dart->ps_prot_slice = APPLE_DART_TEST_PS_SLICE; + regmap_write(dart->ps_prot, APPLE_PS_PROT_ENABLE, 1); + regmap_write(dart->ps_prot, APPLE_PS_PROT_SLICE(APPLE_DART_TEST_PS_SLICE), 0x12); + + writel(1 << 16, dart->regs + DART_PARAMS2); /* one port */ + writel(DART_T8110_TCR_TRANSLATE_ENABLE, dart->regs + DART_TCR(dart, 1)); + writel(dart->hw->ttbr_valid | 0x1234, dart->regs + DART_TTBR(dart, 1, 0)); + writel(DART_T8110_PROTECT_TTBR_TCR, dart->regs + DART_T8110_PROTECT); + writel(DART_T8110_PROTECT_TTBR_TCR, dart->regs + DART_T8110_PROTECT_LOCK); + return dart; +} + +static u32 apple_dart_test_slice(struct apple_dart *dart) +{ + u32 value = 0; + + regmap_read(dart->ps_prot, APPLE_PS_PROT_SLICE(dart->ps_prot_slice), &value); + return value; +} + +static void apple_dart_test_host_power_cycle(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_host_dart(test); + + KUNIT_ASSERT_EQ(test, apple_dart_host_power_off(dart), 0); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dart->host_off)); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dart->commands_gated)); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), DART_T8110_TLB_CMD_POWERDOWN); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x2U); + /* Its registers are not used until power-up. */ + KUNIT_EXPECT_FALSE(test, apple_dart_pm_get(dart)); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_off(dart), -EALREADY); + + KUNIT_ASSERT_EQ(test, apple_dart_host_power_on(dart), 0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dart->host_off)); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dart->commands_gated)); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), DART_T8110_TLB_CMD_V2); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x12U); + KUNIT_EXPECT_TRUE(test, apple_dart_pm_get(dart)); + apple_dart_pm_put(dart); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_on(dart), -EALREADY); +} + +static void apple_dart_test_host_power_lost(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_host_dart(test); + + KUNIT_ASSERT_EQ(test, apple_dart_host_power_off(dart), 0); + /* Power came back with a reset DART: not rebuilt, not used. */ + writel(0, dart->regs + DART_TTBR(dart, 1, 0)); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_on(dart), -EIO); + KUNIT_EXPECT_TRUE(test, READ_ONCE(dart->host_off)); + KUNIT_EXPECT_FALSE(test, apple_dart_pm_get(dart)); +} + +static void apple_dart_test_host_power_refused(struct kunit *test) +{ + struct apple_dart *dart = apple_dart_test_host_dart(test); + + regmap_write(dart->ps_prot, APPLE_PS_PROT_LOCK, 1); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_off(dart), -EPERM); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x12U); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), 0U); + + dart = apple_dart_test_host_dart(test); + /* A stream that stays enabled: the flush and the slice are left alone. */ + writel(BIT(1), dart->regs + DART_T8110_ENABLE_STREAMS); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_off(dart), -EIO); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_DISABLE_STREAMS), BIT(1)); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), 0U); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x12U); + + /* An outstanding transaction that does not drain in time. */ + dart = apple_dart_test_host_dart(test); + writel(1, dart->regs + DART_T8110_OUTSTANDING(0)); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_off(dart), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x12U); + + /* A pending fault is not waited for. */ + dart = apple_dart_test_host_dart(test); + writel(1, dart->regs + DART_T8110_EXCEPTION(0)); + KUNIT_EXPECT_EQ(test, apple_dart_host_power_off(dart), -EIO); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), 0U); + KUNIT_EXPECT_EQ(test, apple_dart_test_slice(dart), 0x12U); +} + +/* + * Generic SID0 DMA while the DCP has the DART powered down: translations, + * flushes and unmaps use the page tables only, and the power-up flushes. + */ +static void apple_dart_test_host_power_tables(struct kunit *test) +{ + const u64 iova = DCP_DART_ALIAS + 3 * DCP_DART_SLICE + 5 * DCP_DART_PAGE; + const unsigned int root = (iova >> 25) & 2047, leaf = (iova >> 14) & 2047; + const u64 pte = (0x12340000ULL >> 4) | 1 | (0xfffULL << 40); + struct apple_dart *dart = apple_dart_test_host_dart(test); + struct iommu_iotlb_gather gather; + struct apple_dart_domain *domain; + struct apple_dart_dcp *dcp; + + dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + domain = kunit_kzalloc(test, sizeof(*domain), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp); + KUNIT_ASSERT_NOT_NULL(test, domain); + dcp->dart = dart; + spin_lock_init(&dcp->lock); + dcp->root = kunit_kzalloc(test, DCP_DART_ENTRIES * sizeof(u64), GFP_KERNEL); + dcp->table[root] = kunit_kzalloc(test, DCP_DART_ENTRIES * sizeof(u64), GFP_KERNEL); + dcp->expected[root] = kunit_kzalloc(test, DCP_DART_ENTRIES * sizeof(u64), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp->root); + KUNIT_ASSERT_NOT_NULL(test, dcp->table[root]); + KUNIT_ASSERT_NOT_NULL(test, dcp->expected[root]); + dcp->root[root] = 0x5670001; + dcp->entry[root] = dcp->root[root]; + dcp->table[root][leaf] = pte; + dcp->expected[root][leaf] = pte; + dart->frontend[0] = dcp; + domain->dcp = dcp; + + KUNIT_ASSERT_EQ(test, apple_dart_host_power_off(dart), 0); + writel(0, dart->regs + DART_T8110_TLB_CMD); /* Any later command shows here. */ + + KUNIT_EXPECT_EQ(test, apple_dart_dcp_translate(dcp, iova + 0x123), + dcp_dart_pte_phys(pte) | 0x123); + KUNIT_EXPECT_EQ(test, apple_dart_dcp_domain_flush(domain), 0); + iommu_iotlb_gather_init(&gather); + KUNIT_EXPECT_EQ(test, apple_dart_dcp_unmap(dcp, iova, DCP_DART_PAGE, 1, &gather), + (size_t)DCP_DART_PAGE); + KUNIT_EXPECT_EQ(test, dcp->table[root][leaf], 0ULL); + KUNIT_EXPECT_EQ(test, dcp->expected[root][leaf], 0ULL); + KUNIT_EXPECT_EQ(test, apple_dart_dcp_translate(dcp, iova), (phys_addr_t)0); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dcp->failed)); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), 0U); + + KUNIT_ASSERT_EQ(test, apple_dart_host_power_on(dart), 0); + KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8110_TLB_CMD), DART_T8110_TLB_CMD_V2); + KUNIT_EXPECT_FALSE(test, READ_ONCE(dart->host_off)); +} +#endif + static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_strict_engines), KUNIT_CASE(apple_dart_test_iommu_specifiers), @@ -989,6 +1177,12 @@ static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_resume_reset_retry), KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8103), KUNIT_CASE(apple_dart_test_resume_retained_bypass_t8110), +#if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) + KUNIT_CASE(apple_dart_test_host_power_cycle), + KUNIT_CASE(apple_dart_test_host_power_lost), + KUNIT_CASE(apple_dart_test_host_power_refused), + KUNIT_CASE(apple_dart_test_host_power_tables), +#endif {} }; diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index d5b60c897a4c2..60c14b261c358 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -190,6 +192,27 @@ #define DART_T8110_ENABLE_STREAMS 0xc00 #define DART_T8110_DISABLE_STREAMS 0xc20 +/* Per-port outstanding transactions; the port counts are in PARAMS2. */ +#define DART_T8110_OUTSTANDING(port) (0x230 + 4 * (port)) +#define DART_T8110_PARAMS2_READ_PORTS GENMASK(18, 16) +#define DART_T8110_PARAMS2_WRITE_PORTS GENMASK(22, 20) +/* How long the power-down and power-up wait for transactions to drain. */ +#define DART_T8110_IDLE_TIMEOUT_US 1000 +/* TLB command of the power-down path: flush everything, also stream table state. */ +#define DART_T8110_TLB_CMD_POWERDOWN (DART_T8110_TLB_CMD_V2 | DART_T8110_TLB_CMD_STT) + +/* + * PS_PROT ("clock protection") bank: one slice per protected DART. An + * enabled slice (bit 4) guards a power-state register of the DART's power + * domain; the native DART power-down clears bit 4 after its streams are + * disabled and flushed, and power-up sets it again. + */ +#define APPLE_PS_PROT_ENABLE 0x20 +#define APPLE_PS_PROT_LOCK 0x24 +#define APPLE_PS_PROT_SLICE(n) (0x100 + 0x40 * (n)) +#define APPLE_PS_PROT_SLICES 64 +#define APPLE_PS_PROT_SLICE_ON BIT(4) + #define DART_T8110_TCR 0x1000 #define DART_T8110_TCR_REMAP GENMASK(11, 8) #define DART_T8110_TCR_REMAP_ENABLE BIT(7) @@ -320,20 +343,38 @@ struct apple_dart { /* A J713 external display DART, borrowed from its external DCP */ bool dispext; /* - * J713 external display DART: its power confirmation across - * sleep, under dispext_pm_lock (a spinlock: only these fields, never + * A display DART whose power can go away under its borrowed streams: + * an external display DART across sleep (its firmware powers it), or an + * internal one powered down by its DCP. Its registers are used only + * with its power confirmed (apple_dart_pm_get()). + */ + bool power_gated; + /* + * Internal display DART: powered down by its DCP (from + * apple_dart_dcp_power_off() until apple_dart_dcp_power_on(); changed + * only under apple_dart_host_lock()), with what the power-up checks + * and restores. + */ + bool host_off; + u32 off_streams, off_protect, off_lock; + /* Its clock-protection slice in the PS_PROT bank, if it has one. */ + struct regmap *ps_prot; + u32 ps_prot_slice, ps_prot_saved; + /* + * Power-gated display DART: its power confirmation across + * sleep, under pm_gate_lock (a spinlock: only these fields, never * across a register access): unverified from suspend until a state * check at a D572(1) of the current generation, which every suspend * and resume moves on; sleeping between suspend and resume; and the - * register uses in flight (apple_dart_dispext_get()), which an - * invalidation waits for (dispext_wq). + * register uses in flight (apple_dart_pm_get()), which an + * invalidation waits for (pm_wq). */ - spinlock_t dispext_pm_lock; - bool dispext_unverified; - bool dispext_sleeping; - u64 dispext_gen; - unsigned int dispext_users; - wait_queue_head_t dispext_wq; + spinlock_t pm_gate_lock; /* the pm_* fields below */ + bool pm_unverified; + bool pm_sleeping; + u64 pm_gen; + unsigned int pm_users; + wait_queue_head_t pm_wq; u32 dispext_protect, dispext_lock; /* PROTECT and PROTECT_LOCK as adopted */ /* Retained radio/display state left in place across suspend-to-idle. */ bool s2idle_kept; @@ -1576,10 +1617,10 @@ static int apple_dart_dcp_invalidate_tlb_gated(struct apple_dart_stream_map *str { int ret; - if (!apple_dart_dispext_get(stream_map->dart)) + if (!apple_dart_pm_get(stream_map->dart)) return -EAGAIN; ret = apple_dart_dcp_invalidate_tlb(stream_map); - apple_dart_dispext_put(stream_map->dart); + apple_dart_pm_put(stream_map->dart); return ret; } @@ -2029,17 +2070,25 @@ static int __apple_dart_dcp_domain_flush(struct apple_dart_domain *domain); /* * Borrowed DCP domains have no io-pgtable; their flush ignores the range. * On an external display DART only with its power confirmed: -EAGAIN - * (nothing retired) from sleep until its next confirmed power-up. + * (nothing retired) from sleep until its next confirmed power-up. An + * internal display DART that its DCP has powered down needs none: nothing + * is cached, and its power-up flushes every update made before it (they + * hold the stream's lock, which the power-up takes; apple_dart_host_lock()). + * A power-up that completed meanwhile confirmed the power in the same step. */ static int apple_dart_dcp_domain_flush(struct apple_dart_domain *domain) { struct apple_dart *dart = domain->dcp->dart; int ret; - if (!apple_dart_dispext_get(dart)) - return -EAGAIN; + if (!apple_dart_pm_get(dart)) { + if (READ_ONCE(dart->host_off)) + return 0; + if (!dart->ps_prot || !apple_dart_pm_get(dart)) + return -EAGAIN; + } ret = __apple_dart_dcp_domain_flush(domain); - apple_dart_dispext_put(dart); + apple_dart_pm_put(dart); return ret; } @@ -2785,11 +2834,11 @@ static struct iommu_device *apple_dart_probe_device(struct device *dev) int ret = -EINVAL; /* An external display DART's registers only inside a use. */ - if (borrowed && !apple_dart_dispext_get(borrowed->dart)) { + if (borrowed && !apple_dart_pm_get(borrowed->dart)) { ret = -EAGAIN; } else if (borrowed) { ret = apple_dart_dcp_validate(borrowed, dev); - apple_dart_dispext_put(borrowed->dart); + apple_dart_pm_put(borrowed->dart); } if (ret) return ERR_PTR(ret); @@ -3054,7 +3103,7 @@ static int apple_dart_of_xlate(struct device *dev, #if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) /* * The check, inside a use of the DART's registers (on an external display - * DART, apple_dart_dispext_get() by the caller). + * DART, apple_dart_pm_get() by the caller). */ static int __apple_dart_dcp_check(struct device *dev) { @@ -3126,10 +3175,10 @@ int apple_dart_dcp_check(struct device *dev) if (!domain || domain->owner != &apple_dart_iommu_ops || !to_dart_domain(domain)->dcp) return __apple_dart_dcp_check(dev); dart = to_dart_domain(domain)->dcp->dart; - if (!apple_dart_dispext_get(dart)) + if (!apple_dart_pm_get(dart)) return -EAGAIN; ret = __apple_dart_dcp_check(dev); - apple_dart_dispext_put(dart); + apple_dart_pm_put(dart); return ret; } EXPORT_SYMBOL_GPL(apple_dart_dcp_check); @@ -4261,8 +4310,21 @@ static int apple_dart_probe(struct platform_device *pdev) spin_lock_init(&dart->managed_lock); dart->preserve_display = j713_display || j713_dcpext || j713_dispext; dart->dispext = j713_dispext; - spin_lock_init(&dart->dispext_pm_lock); - init_waitqueue_head(&dart->dispext_wq); + dart->power_gated = j713_dispext; + if (j713_display && of_property_present(dev->of_node, "apple,clock-protection")) { + /* Its DCP can power it down: see apple_dart_dcp_power_off(). */ + dart->ps_prot = syscon_regmap_lookup_by_phandle_args(dev->of_node, + "apple,clock-protection", + 1, &dart->ps_prot_slice); + if (IS_ERR(dart->ps_prot)) + return dev_err_probe(dev, PTR_ERR(dart->ps_prot), + "clock-protection bank\n"); + if (dart->ps_prot_slice >= APPLE_PS_PROT_SLICES) + return -EINVAL; + dart->power_gated = true; + } + spin_lock_init(&dart->pm_gate_lock); + init_waitqueue_head(&dart->pm_wq); if (j713_dcpext) dart->dcp_instance = &dcp_adt_external[res->start == 0x48a340000ULL]; else if (j713_dispext) @@ -5061,7 +5123,7 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) return -EBUSY; if (apple_dart_has_borrowed(dart)) - apple_dart_dispext_invalidate(dart, true); + apple_dart_pm_invalidate(dart, true); return 0; } /* @@ -5180,16 +5242,19 @@ static __maybe_unused int apple_dart_resume(struct device *dev) /* No reset or restore: see apple_dart_suspend(). */ if (dart->cold_ane) return 0; + /* Powered down by its DCP, which restores it at power-up. */ + if (READ_ONCE(dart->host_off)) + return 0; /* * An adopted external display DART may be off now, its display with it, * and nothing confirms otherwise: no register of it is read or written - * (apple_dart_dispext_unconfirmed()) until its firmware powers it up + * (apple_dart_pm_unconfirmed()) until its firmware powers it up * again; the state check then (apple_dart_dcp_dispext_state()) runs * first: kept, it goes on; lost, it is retired, saying so. */ if (dart->dispext) { if (apple_dart_has_borrowed(dart)) { - apple_dart_dispext_invalidate(dart, false); + apple_dart_pm_invalidate(dart, false); dev_dbg(dev, "external display DART unconfirmed after sleep\n"); } return 0; @@ -5270,6 +5335,259 @@ static const struct of_device_id apple_dart_of_match[] = { MODULE_DEVICE_TABLE(of, apple_dart_of_match); #if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) +/* The internal display DART of a DCP consumer (its firmware or display stream). */ +static struct apple_dart *apple_dart_dcp_consumer(struct device *dev) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct apple_dart_domain *dd; + + if (!domain || domain->owner != &apple_dart_iommu_ops) + return NULL; + dd = to_dart_domain(domain); + if (!dd->dcp || dd->dcp->owner != dev || dd->dcp->dart->dispext || + !dd->dcp->dart->ps_prot) + return NULL; + return dd->dcp->dart; +} + +static int apple_dart_tlb_command(struct apple_dart *dart, u32 command) +{ + u32 status; + + writel(command, dart->regs + DART_T8110_TLB_CMD); + return readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_STREAM_COMMAND_BUSY_TIMEOUT); +} + +/* + * Nothing in flight: no fault (-EIO), and no TLB command or outstanding + * transaction (-EBUSY, which can drain). + */ +static int apple_dart_idle(struct apple_dart *dart) +{ + u32 params = readl(dart->regs + DART_PARAMS2); + unsigned int ports = max(FIELD_GET(DART_T8110_PARAMS2_READ_PORTS, params), + FIELD_GET(DART_T8110_PARAMS2_WRITE_PORTS, params)); + + if (!ports || ports > 4 || apple_dart_fault_pending(dart)) + return -EIO; + if (readl(dart->regs + DART_T8110_TLB_CMD) & DART_T8110_TLB_CMD_BUSY) + return -EBUSY; + for (unsigned int port = 0; port < ports; port++) + if (readl(dart->regs + DART_T8110_OUTSTANDING(port))) + return -EBUSY; + return 0; +} + +/* Idle, giving transactions and a command just issued time to drain. */ +static int apple_dart_wait_idle(struct apple_dart *dart) +{ + int ret, idle; + + ret = read_poll_timeout_atomic(apple_dart_idle, idle, idle != -EBUSY, 1, + DART_T8110_IDLE_TIMEOUT_US, false, dart); + return ret ?: idle; +} + +/* Its PS_PROT bank enabled and not locked: its slice can be switched. */ +static int apple_dart_ps_prot_usable(struct apple_dart *dart) +{ + u32 enable, lock; + int ret; + + ret = regmap_read(dart->ps_prot, APPLE_PS_PROT_ENABLE, &enable) ?: + regmap_read(dart->ps_prot, APPLE_PS_PROT_LOCK, &lock); + if (!ret && (!(enable & BIT(0)) || (lock & BIT(0)))) + ret = -EPERM; + return ret; +} + +static int apple_dart_ps_prot_set(struct apple_dart *dart, u32 value) +{ + unsigned int reg = APPLE_PS_PROT_SLICE(dart->ps_prot_slice), now; + int ret; + + ret = regmap_write(dart->ps_prot, reg, value); + if (!ret) + ret = regmap_read(dart->ps_prot, reg, &now); + return ret ?: now == value ? 0 : -EIO; +} + +/* + * Whether apple_dart_dcp_power_off() can run for @dev's DART, changing + * nothing: its DCP asks before the first step of its power-down. + */ +int apple_dart_dcp_power_check(struct device *dev) +{ + struct apple_dart *dart = apple_dart_dcp_consumer(dev); + struct apple_dart_dcp *streams[3]; + + if (!dart) + return -ENODEV; + if (READ_ONCE(dart->host_off)) + return -EALREADY; + streams[0] = dart->dcp; + streams[1] = dart->frontend[0]; + streams[2] = dart->frontend[1]; + for (unsigned int i = 0; i < ARRAY_SIZE(streams); i++) + if (streams[i] && READ_ONCE(streams[i]->failed)) + return -EIO; + return apple_dart_ps_prot_usable(dart); +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_power_check); + +/* + * J713 internal display DART, before its DCP powers the display domains + * down (its DCP CPU stopped, the DART's masters idle): as the native DART + * power-down does, disable its enabled streams, flush with the power-down + * TLB command and clear its clock-protection slice. From here until + * apple_dart_dcp_power_on() its registers are not used: its commands are + * gated, its generic DMA streams update their page tables in memory only + * (apple_dart_host_lock()) and system sleep leaves it alone. On failure its + * borrowed streams are retired. + * @dev is a consumer: the DCP (firmware DART) or its PIODMA (display DART). + */ +static int apple_dart_host_power_off(struct apple_dart *dart) +{ + unsigned long flags; + u32 enabled; + const char *why; + int ret; + + if (READ_ONCE(dart->host_off)) + return -EALREADY; + /* + * Page-table updates stay in memory from now on; then no use of its + * registers either, and the uses in flight are waited for. + */ + apple_dart_host_lock(dart, &flags); + WRITE_ONCE(dart->host_off, true); + apple_dart_host_unlock(dart, flags); + apple_dart_pm_invalidate(dart, false); + apple_dart_quiesce_commands(dart->dev); + + why = "not idle"; + ret = apple_dart_wait_idle(dart); + if (ret) + goto fail; + why = "clock protection"; + ret = apple_dart_ps_prot_usable(dart) ?: + regmap_read(dart->ps_prot, APPLE_PS_PROT_SLICE(dart->ps_prot_slice), + &dart->ps_prot_saved); + if (ret) + goto fail; + + /* What power-up checks: the translations and their lock, and the streams. */ + apple_dart_save_state(dart); + dart->off_protect = readl(dart->regs + DART_T8110_PROTECT); + dart->off_lock = readl(dart->regs + DART_T8110_PROTECT_LOCK); + dart->off_streams = readl(dart->regs + DART_T8110_ENABLE_STREAMS); + + why = "stream disable"; + writel(dart->off_streams, dart->regs + DART_T8110_DISABLE_STREAMS); + enabled = readl(dart->regs + DART_T8110_ENABLE_STREAMS); + ret = enabled ? -EIO : 0; + if (ret) + goto fail; + why = "power-down flush"; + ret = apple_dart_tlb_command(dart, DART_T8110_TLB_CMD_POWERDOWN) ?: + apple_dart_wait_idle(dart); + if (ret) + goto fail; + if (dart->ps_prot_saved & APPLE_PS_PROT_SLICE_ON) { + why = "clock protection off"; + ret = apple_dart_ps_prot_set(dart, dart->ps_prot_saved & ~APPLE_PS_PROT_SLICE_ON); + if (ret) + goto fail; + } + dev_dbg(dart->dev, "powered down: streams %08x, clock protection %08x\n", + dart->off_streams, dart->ps_prot_saved); + return 0; +fail: + apple_dart_fail_borrowed(dart, "power-down: %s (%d)", why, ret); + return ret; +} + +int apple_dart_dcp_power_off(struct device *dev) +{ + struct apple_dart *dart = apple_dart_dcp_consumer(dev); + + return dart ? apple_dart_host_power_off(dart) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_power_off); + +/* + * After its DCP powered the display domains up again, before the DCP CPU + * runs: restore the clock-protection slice, check that the DART kept its + * translations and their lock (it is not rebuilt), flush, and enable the + * streams that were enabled at power-down. Its registers are then usable + * again. On failure its borrowed streams are retired. + */ +static int apple_dart_host_power_on(struct apple_dart *dart) +{ + unsigned long flags; + const char *why; + int ret; + + if (!READ_ONCE(dart->host_off)) + return -EALREADY; + why = "clock protection on"; + ret = apple_dart_ps_prot_set(dart, dart->ps_prot_saved); + if (ret) + goto fail; + why = "state lost"; + ret = -EIO; + if (readl(dart->regs + DART_T8110_ENABLE_STREAMS) || + readl(dart->regs + DART_T8110_PROTECT) != dart->off_protect || + readl(dart->regs + DART_T8110_PROTECT_LOCK) != dart->off_lock || + !apple_dart_state_retained(dart)) + goto fail; + why = "commands"; + ret = apple_dart_resume_commands(dart->dev); + if (ret) + goto fail; + + /* + * The flush, the streams and the confirmed power are one step for the + * streams' page-table updates (apple_dart_host_lock()): the flush covers + * every update made while the DART was down, and every later update + * finds it powered. + */ + apple_dart_host_lock(dart, &flags); + why = "flush"; + ret = apple_dart_wait_idle(dart) ?: + apple_dart_tlb_command(dart, DART_T8110_TLB_CMD_V2); + if (!ret) { + why = "stream enable"; + writel(dart->off_streams, dart->regs + DART_T8110_ENABLE_STREAMS); + if (readl(dart->regs + DART_T8110_ENABLE_STREAMS) != dart->off_streams) + ret = -EIO; + } + if (!ret) { + WRITE_ONCE(dart->host_off, false); + spin_lock(&dart->pm_gate_lock); + dart->pm_unverified = false; + spin_unlock(&dart->pm_gate_lock); + } + apple_dart_host_unlock(dart, flags); + if (ret) + goto fail; + dev_dbg(dart->dev, "powered up: state kept, streams %08x\n", dart->off_streams); + return 0; +fail: + apple_dart_fail_borrowed(dart, "power-up: %s (%d)", why, ret); + return ret; +} + +int apple_dart_dcp_power_on(struct device *dev) +{ + struct apple_dart *dart = apple_dart_dcp_consumer(dev); + + return dart ? apple_dart_host_power_on(dart) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_power_on); + /* * J713 external DCP: whether an adopted external display DART kept its * state, its power on (the caller's to know): the TTBR/TCR lock (PROTECT @@ -5290,16 +5608,16 @@ static int apple_dart_dispext_verify(struct apple_dart *dart, const char *when) * not while asleep (between suspend and resume nothing confirms its * power), and an invalidation meanwhile waits for it to end. */ - spin_lock_irqsave(&dart->dispext_pm_lock, flags); - if (dart->dispext_sleeping) { - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + spin_lock_irqsave(&dart->pm_gate_lock, flags); + if (dart->pm_sleeping) { + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); dev_dbg(dart->dev, "external display DART %s during sleep: not checked\n", when); return -EAGAIN; } - gen = dart->dispext_gen; - was = dart->dispext_unverified; - dart->dispext_users++; - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + gen = dart->pm_gen; + was = dart->pm_unverified; + dart->pm_users++; + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); protect = readl(dart->regs + DART_T8110_PROTECT); lock = readl(dart->regs + DART_T8110_PROTECT_LOCK); @@ -5319,14 +5637,14 @@ static int apple_dart_dispext_verify(struct apple_dart *dart, const char *when) kept &= (protect & lock & DART_T8110_PROTECT_TTBR_TCR) == (dart->dispext_protect & dart->dispext_lock & DART_T8110_PROTECT_TTBR_TCR); - spin_lock_irqsave(&dart->dispext_pm_lock, flags); + spin_lock_irqsave(&dart->pm_gate_lock, flags); /* Only a check of the current generation confirms: a sleep since voids it. */ - current_gen = gen == dart->dispext_gen; + current_gen = gen == dart->pm_gen; if (kept && current_gen) - dart->dispext_unverified = false; - if (!--dart->dispext_users) - wake_up_all(&dart->dispext_wq); - spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + dart->pm_unverified = false; + if (!--dart->pm_users) + wake_up_all(&dart->pm_wq); + spin_unlock_irqrestore(&dart->pm_gate_lock, flags); dev_dbg(dart->dev, "external display DART %s: protect %08x lock %08x (adopted %08x/%08x): state %s%s%s\n", when, protect, lock, dart->dispext_protect, dart->dispext_lock, diff --git a/drivers/pmdomain/apple/pmc-test.c b/drivers/pmdomain/apple/pmc-test.c index d563a178dc0e8..4e7f19d47317a 100644 --- a/drivers/pmdomain/apple/pmc-test.c +++ b/drivers/pmdomain/apple/pmc-test.c @@ -355,6 +355,64 @@ static void apple_pmc_test_interface_stuck(struct kunit *test) test->priv = NULL; } +/* + * The internal display's consumer-switched interface (BWR and boost bit 0): + * off clears its bits only, on gives back the values they had, whether the + * boot loader left both set or only the boost enable. + */ +static void apple_pmc_test_interface_consumer(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + struct apple_pmc_domain dom = { + .pmc = pmc, .interface = true, .bwr_bit = 0, .boost_bit = 0, + .consumer_switched = true, + }; + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8132; + + pmc->hw = hw; + /* As Linux finds them: the boost enable only, next to dispext0's. */ + writel(BIT(0) | BIT(1), pmc->rails + hw->boost_enable); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_consumer_set(&dom, true), -EALREADY); + KUNIT_ASSERT_EQ(test, apple_pmc_interface_consumer_set(&dom, false), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), 0U); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(1)); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_consumer_set(&dom, false), -EALREADY); + KUNIT_ASSERT_EQ(test, apple_pmc_interface_consumer_set(&dom, true), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), 0U); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(0) | BIT(1)); + + /* As macOS keeps them while the display is on: both set. */ + writel(BIT(0), pmc->rails + hw->bwr_enable); + KUNIT_ASSERT_EQ(test, apple_pmc_interface_consumer_set(&dom, false), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), 0U); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(1)); + KUNIT_ASSERT_EQ(test, apple_pmc_interface_consumer_set(&dom, true), 0); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->bwr_enable), BIT(0)); + KUNIT_EXPECT_EQ(test, readl(pmc->rails + hw->boost_enable), BIT(0) | BIT(1)); +} + +/* A BWR enable that does not clear: the boost enable goes back, still on. */ +static void apple_pmc_test_interface_consumer_stuck(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + struct apple_pmc_domain dom = { + .pmc = pmc, .interface = true, .bwr_bit = 0, .boost_bit = 0, + .consumer_switched = true, + }; + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8132; + struct apple_pmc_test_stuck stuck = { .offset = hw->bwr_enable, .value = BIT(0) }; + + pmc->hw = hw; + writel(BIT(0), pmc->rails + hw->boost_enable); + test->priv = &stuck; + kunit_activate_static_stub(test, apple_pmc_interface_read, apple_pmc_test_stuck_read); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_consumer_set(&dom, false), -EIO); + KUNIT_EXPECT_FALSE(test, dom.consumer_off); + KUNIT_EXPECT_EQ(test, apple_pmc_interface_read(pmc, hw->boost_enable), BIT(0)); + kunit_deactivate_static_stub(test, apple_pmc_interface_read); + test->priv = NULL; +} + static void apple_pmc_test_genpd_remove(void *genpd) { pm_genpd_remove(genpd); @@ -497,6 +555,8 @@ static struct kunit_case apple_pmc_test_cases[] = { KUNIT_CASE(apple_pmc_test_floor_pending_timeout), KUNIT_CASE(apple_pmc_test_interface), KUNIT_CASE(apple_pmc_test_interface_stuck), + KUNIT_CASE(apple_pmc_test_interface_consumer), + KUNIT_CASE(apple_pmc_test_interface_consumer_stuck), KUNIT_CASE(apple_pmc_test_interface_genpd), KUNIT_CASE(apple_pmc_test_interface_boot_state), {} diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 3765e9a775ff7..40392c9d46603 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -152,6 +152,10 @@ struct apple_pmc { * @interface: the domain is a display pipe's BWR and boost interface enables * @bwr_bit: with @interface, the bit in the BWR interface enable word * @boost_bit: with @interface, the bit in the boost interface enable word + * @consumer_switched: with @interface, not a genpd domain: its consumer clears + * and restores its bits (apple_pmc_interface_consumer_off()) + * @consumer_off: with @consumer_switched, cleared by its consumer, and the + * bits it had then (@consumer_bwr, @consumer_boost); protected by pmc->lock */ struct apple_pmc_domain { struct generic_pm_domain genpd; @@ -168,6 +172,10 @@ struct apple_pmc_domain { bool interface; unsigned int bwr_bit; unsigned int boost_bit; + bool consumer_switched; + bool consumer_off; + bool consumer_bwr; + bool consumer_boost; bool provider; }; @@ -766,6 +774,107 @@ static int apple_pmc_interface_set(struct apple_pmc_domain *dom, bool on) return ret; } +/* + * A consumer-switched interface: off clears the boost, then the BWR enable, + * as a pipe's power-down does, after saving both; on gives both back what + * they had then. If the BWR enable does not take the change on the way off, + * the boost enable gets its old value back. Nothing is retried. + */ +static int apple_pmc_interface_consumer_set(struct apple_pmc_domain *dom, bool on) +{ + struct apple_pmc *pmc = dom->pmc; + const struct apple_pmc_hw *hw = pmc->hw; + unsigned long flags; + int ret; + + raw_spin_lock_irqsave(&pmc->lock, flags); + if (on != dom->consumer_off) { + ret = -EALREADY; + } else if (on) { + ret = apple_pmc_interface_update(pmc, hw->boost_enable, dom->boost_bit, + dom->consumer_boost); + if (!ret) + ret = apple_pmc_interface_update(pmc, hw->bwr_enable, dom->bwr_bit, + dom->consumer_bwr); + if (!ret) + dom->consumer_off = false; + } else { + dom->consumer_bwr = apple_pmc_interface_read(pmc, hw->bwr_enable) & + BIT(dom->bwr_bit); + dom->consumer_boost = apple_pmc_interface_read(pmc, hw->boost_enable) & + BIT(dom->boost_bit); + ret = apple_pmc_interface_update(pmc, hw->boost_enable, dom->boost_bit, false); + if (!ret) { + ret = apple_pmc_interface_update(pmc, hw->bwr_enable, dom->bwr_bit, false); + if (ret) + apple_pmc_interface_update(pmc, hw->boost_enable, dom->boost_bit, + dom->consumer_boost); + } + if (!ret) + dom->consumer_off = true; + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + return ret; +} + +static struct platform_driver apple_pmc_driver; + +/* The consumer-switched interface named by @consumer's apple,pmc-interface. */ +static struct apple_pmc_domain *apple_pmc_consumer_interface(struct device *consumer) +{ + struct device_node *np = of_parse_phandle(consumer->of_node, "apple,pmc-interface", 0); + struct apple_pmc_domain *dom, *found = NULL; + struct platform_device *pdev; + struct apple_pmc *pmc = NULL; + + if (!np) + return NULL; + pdev = of_find_device_by_node(np->parent); + if (pdev && pdev->dev.driver == &apple_pmc_driver.driver) + pmc = platform_get_drvdata(pdev); + if (pmc) + list_for_each_entry(dom, &pmc->domains, list) + if (dom->np == np && dom->consumer_switched) + found = dom; + if (pdev) + put_device(&pdev->dev); + of_node_put(np); + return found; +} + +/* Whether apple_pmc_interface_consumer_off() can run for @consumer now, changing nothing. */ +int apple_pmc_interface_consumer_check(struct device *consumer) +{ + struct apple_pmc_domain *dom = apple_pmc_consumer_interface(consumer); + unsigned long flags; + int ret; + + if (!dom) + return -ENODEV; + raw_spin_lock_irqsave(&dom->pmc->lock, flags); + ret = dom->consumer_off ? -EALREADY : 0; + raw_spin_unlock_irqrestore(&dom->pmc->lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmc_interface_consumer_check); + +int apple_pmc_interface_consumer_off(struct device *consumer) +{ + struct apple_pmc_domain *dom = apple_pmc_consumer_interface(consumer); + + return dom ? apple_pmc_interface_consumer_set(dom, false) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_pmc_interface_consumer_off); + +int apple_pmc_interface_consumer_on(struct device *consumer) +{ + struct apple_pmc_domain *dom = apple_pmc_consumer_interface(consumer); + + return dom ? apple_pmc_interface_consumer_set(dom, true) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_pmc_interface_consumer_on); + static int apple_pmc_domain_power_on(struct generic_pm_domain *genpd) { struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); @@ -794,6 +903,8 @@ static int apple_pmc_domain_power_off(struct generic_pm_domain *genpd) static void apple_pmc_domain_free(struct apple_pmc_domain *dom) { + if (dom->consumer_switched) + goto free; if (dom->provider) of_genpd_del_provider(dom->np); if (pm_genpd_remove(&dom->genpd)) { @@ -802,6 +913,7 @@ static void apple_pmc_domain_free(struct apple_pmc_domain *dom) dom->genpd.name); return; } +free: of_node_put(dom->np); kfree(dom->genpd.name); kfree(dom); @@ -1011,6 +1123,15 @@ static int apple_pmc_interface_add(struct apple_pmc *pmc, struct device_node *np return -ENOMEM; } + /* Its consumer clears and restores its bits: no genpd domain. */ + dom->consumer_switched = of_property_read_bool(np, "apple,consumer-switched"); + if (dom->consumer_switched) { + list_add_tail(&dom->list, &pmc->domains); + pmc->bwr_bits |= BIT(bwr); + pmc->boost_bits |= BIT(boost); + return 0; + } + ret = apple_pmc_interface_init(pmc, dom); if (ret) { of_node_put(dom->np); @@ -1280,7 +1401,9 @@ static int apple_pmc_probe(struct platform_device *pdev) } for_each_available_child_of_node(np, child) { - if (!of_property_present(child, "#power-domain-cells")) + /* A consumer-switched interface is no power domain, but still ours. */ + if (!of_property_present(child, "#power-domain-cells") && + !of_property_read_bool(child, "apple,consumer-switched")) continue; if (of_property_present(child, "apple,rail")) ret = apple_pmc_floor_add(pmc, child); diff --git a/drivers/pmdomain/apple/pmgr-pwrstate-test.c b/drivers/pmdomain/apple/pmgr-pwrstate-test.c index 67698ccfc74aa..66fb065523dee 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate-test.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate-test.c @@ -357,6 +357,193 @@ static void pmgr_ps_power_on_no_reset_test(struct kunit *test) KUNIT_EXPECT_EQ(test, r->reset_writes, 0U); } +/* + * Consumer-switched states: a state word at offset 0 whose actual state + * follows the target (the history bits are set by a power-gate and cleared + * by writing them), retention registers as plain words, and a write log. + */ +struct pmgr_ps_test_bank { + u32 word[0x30000 / 4]; + bool stuck; /* the actual state no longer follows the target */ + bool ignore_retention; /* retention writes are dropped */ + unsigned int nr_log; + struct { u32 reg, val; } log[16]; +}; + +static int pmgr_ps_test_bank_read(void *ctx, unsigned int reg, unsigned int *val) +{ + struct pmgr_ps_test_bank *b = ctx; + + *val = b->word[reg / 4]; + return 0; +} + +static int pmgr_ps_test_bank_write(void *ctx, unsigned int reg, unsigned int val) +{ + struct pmgr_ps_test_bank *b = ctx; + u32 old = b->word[reg / 4], target, cleared; + + if (b->nr_log < ARRAY_SIZE(b->log)) { + b->log[b->nr_log].reg = reg; + b->log[b->nr_log++].val = val; + } + if (reg) { + if (!b->ignore_retention) + b->word[reg / 4] = val; + return 0; + } + target = FIELD_GET(APPLE_PMGR_PS_TARGET, val); + cleared = val & APPLE_PMGR_FLAGS; /* written as 1: cleared */ + val &= ~(APPLE_PMGR_PS_ACTUAL | APPLE_PMGR_FLAGS); + val |= old & APPLE_PMGR_FLAGS & ~cleared; + val |= FIELD_PREP(APPLE_PMGR_PS_ACTUAL, + b->stuck ? FIELD_GET(APPLE_PMGR_PS_ACTUAL, old) : target); + if (!b->stuck && target == APPLE_PMGR_PS_PWRGATE) + val |= APPLE_PMGR_FLAGS; + b->word[0] = val; + return 0; +} + +static const struct regmap_config pmgr_ps_test_bank_regmap = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .max_register = 0x30000 - 4, + .reg_read = pmgr_ps_test_bank_read, + .reg_write = pmgr_ps_test_bank_write, + .fast_io = true, +}; + +/* The J713 DCP CPU state: AUTO on, with its two retention workaround words. */ +static struct apple_pmgr_ps *pmgr_ps_test_consumer(struct kunit *test, u32 word, + struct pmgr_ps_test_bank **bank) +{ + static atomic_t instance = ATOMIC_INIT(0); + struct pmgr_ps_test_bank *b; + struct apple_pmgr_ps *ps; + struct device *dev; + char name[32]; + + b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, b); + b->word[0] = word; + b->word[0x2504c / 4] = 0; + b->word[0x25070 / 4] = 1; + snprintf(name, sizeof(name), "apple-pmgr-consumer-%d", atomic_inc_return(&instance)); + dev = kunit_device_register(test, name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + ps = kunit_kzalloc(test, sizeof(*ps), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ps); + ps->dev = dev; + ps->genpd.name = "disp-cpu"; + ps->firmware_owned = true; + ps->consumer_switched = true; + ps->nr_retention = 2; + ps->retention[0].offset = 0x2504c; + ps->retention[0].mask = 3; + ps->retention[0].value = 3; + ps->retention[1].offset = 0x25070; + ps->retention[1].mask = 1; + ps->retention[1].value = 0; + ps->regmap = devm_regmap_init(dev, NULL, b, &pmgr_ps_test_bank_regmap); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ps->regmap); + *bank = b; + return ps; +} + +static void pmgr_ps_test_expect_log(struct kunit *test, struct pmgr_ps_test_bank *b, + const u32 (*want)[2], unsigned int n) +{ + KUNIT_ASSERT_EQ(test, b->nr_log, n); + for (unsigned int i = 0; i < n; i++) { + KUNIT_EXPECT_EQ_MSG(test, b->log[i].reg, want[i][0], "write %u", i); + KUNIT_EXPECT_EQ_MSG(test, b->log[i].val, want[i][1], "write %u", i); + } + b->nr_log = 0; +} + +/* The native J713 DCP CPU writes, as captured on macOS, in order. */ +static void pmgr_ps_consumer_native_test(struct kunit *test) +{ + static const u32 off[][2] = { + { 0, 0x0f0000ff }, { 0x2504c, 3 }, { 0x25070, 0 }, { 0, 0x0f0003f0 }, + }; + static const u32 on[][2] = { + { 0, 0x0f00000f }, { 0x2504c, 0 }, { 0x25070, 1 }, { 0, 0x1f00000f }, + }; + struct pmgr_ps_test_bank *b; + struct apple_pmgr_ps *ps = pmgr_ps_test_consumer(test, 0x1f0003ff, &b); + + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), 0); + pmgr_ps_test_expect_log(test, b, off, ARRAY_SIZE(off)); + KUNIT_EXPECT_EQ(test, b->word[0] & (APPLE_PMGR_PS_TARGET | APPLE_PMGR_PS_ACTUAL), 0U); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), -EALREADY); + + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), 0); + pmgr_ps_test_expect_log(test, b, on, ARRAY_SIZE(on)); + KUNIT_EXPECT_EQ(test, b->word[0] & ~APPLE_PMGR_FLAGS, 0x1f0000ffU); + KUNIT_EXPECT_EQ(test, b->word[0x2504c / 4], 0U); + KUNIT_EXPECT_EQ(test, b->word[0x25070 / 4], 1U); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), -EALREADY); +} + +/* Retention bits outside the workaround's mask that change while off are kept. */ +static void pmgr_ps_consumer_retention_mask_test(struct kunit *test) +{ + struct pmgr_ps_test_bank *b; + struct apple_pmgr_ps *ps = pmgr_ps_test_consumer(test, 0x1f0003ff, &b); + + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), 0); + b->word[0x2504c / 4] |= 0x100; + b->word[0x25070 / 4] |= 0x10; + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), 0); + KUNIT_EXPECT_EQ(test, b->word[0x2504c / 4], 0x100U); + KUNIT_EXPECT_EQ(test, b->word[0x25070 / 4], 0x11U); +} + +/* FE and SYS: no AUTO and no retention; other bits are kept. */ +static void pmgr_ps_consumer_plain_test(struct kunit *test) +{ + static const u32 off[][2] = { { 0, 0x23f0 } }; + static const u32 on[][2] = { { 0, 0x200f } }; + struct pmgr_ps_test_bank *b; + struct apple_pmgr_ps *ps = pmgr_ps_test_consumer(test, 0x20ff, &b); + + ps->nr_retention = 0; + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), 0); + pmgr_ps_test_expect_log(test, b, off, ARRAY_SIZE(off)); + KUNIT_ASSERT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), 0); + pmgr_ps_test_expect_log(test, b, on, ARRAY_SIZE(on)); +} + +static void pmgr_ps_consumer_refused_test(struct kunit *test) +{ + struct pmgr_ps_test_bank *b; + struct apple_pmgr_ps *ps; + + /* Not fully on: left alone, and not failed. */ + ps = pmgr_ps_test_consumer(test, 0x1f000044, &b); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), -EBUSY); + KUNIT_EXPECT_EQ(test, b->nr_log, 0U); + KUNIT_EXPECT_FALSE(test, ps->consumer_failed); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), -EALREADY); + + /* The actual state does not follow: failed, and stays failed. */ + ps = pmgr_ps_test_consumer(test, 0x1f0003ff, &b); + b->stuck = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, ps->consumer_failed); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_on(ps), -EIO); + + /* A retention register that does not take the value: failed before power-off. */ + ps = pmgr_ps_test_consumer(test, 0x1f0003ff, &b); + b->ignore_retention = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_consumer_state_off(ps), -EIO); + KUNIT_EXPECT_TRUE(test, ps->consumer_failed); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_TARGET, b->word[0]), + APPLE_PMGR_PS_ACTIVE); +} + static struct kunit_case pmgr_ps_test_cases[] = { KUNIT_CASE(pmgr_ps_power_on_test), KUNIT_CASE(pmgr_ps_power_off_undo_test), @@ -372,6 +559,10 @@ static struct kunit_case pmgr_ps_test_cases[] = { KUNIT_CASE(pmgr_ps_power_on_latch_sleep_test), KUNIT_CASE(pmgr_ps_power_on_latch_sticky_test), KUNIT_CASE(pmgr_ps_power_on_no_reset_test), + KUNIT_CASE(pmgr_ps_consumer_native_test), + KUNIT_CASE(pmgr_ps_consumer_retention_mask_test), + KUNIT_CASE(pmgr_ps_consumer_plain_test), + KUNIT_CASE(pmgr_ps_consumer_refused_test), {} }; diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 688842390a6a2..d6b0cc28b6c2c 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -41,6 +41,14 @@ #define APPLE_PMGR_PS_PWRGATE 0x0 #define APPLE_PMGR_PS_SET_TIMEOUT 100 +#define APPLE_PMGR_PS_CONSUMER_TIMEOUT 1000 +#define APPLE_PMGR_PS_MAX_RETENTION 4 +#define APPLE_PMGR_PS_MAX_CHAIN 4 + +#define APPLE_PMGR_PS_LEVELS (APPLE_PMGR_PS_TARGET | APPLE_PMGR_PS_ACTUAL) +/* Target and actual state both on (or both power gated). */ +#define APPLE_PMGR_PS_ALL_ON (FIELD_PREP(APPLE_PMGR_PS_TARGET, APPLE_PMGR_PS_ACTIVE) | \ + FIELD_PREP(APPLE_PMGR_PS_ACTUAL, APPLE_PMGR_PS_ACTIVE)) #define APPLE_PMGR_PS_PARENT_TIMEOUT 10000 #define APPLE_PMGR_RESET_TIME 1 #define APPLE_PMGR_T8132_RESET_TIME 10 @@ -67,6 +75,20 @@ struct apple_pmgr_ps { bool strict_state; bool wait_parent; u32 reset_time_us; + /* + * apple,consumer-switched: firmware-owned, but powered off and on by its + * consumer (apple_pmgr_ps_consumer_off()). The state word before + * power-off, and the retention workaround registers with their values + * before power-off. A failed transition leaves it failed until reboot. + */ + bool consumer_switched; + bool consumer_off; + bool consumer_failed; + u32 consumer_saved; + unsigned int nr_retention; + struct { + u32 offset, mask, value, saved; + } retention[APPLE_PMGR_PS_MAX_RETENTION]; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -649,6 +671,28 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) * vote and never read, reset or change the hardware state. */ ps->firmware_owned = of_property_read_bool(node, "apple,firmware-owned"); + ps->consumer_switched = of_property_read_bool(node, "apple,consumer-switched"); + if (ps->consumer_switched) { + int n = of_property_count_u32_elems(node, "apple,retention-workaround"); + + if (!ps->firmware_owned || + (n > 0 && (n % 3 || n / 3 > APPLE_PMGR_PS_MAX_RETENTION))) + return dev_err_probe(dev, -EINVAL, "bad consumer-switched policy\n"); + for (int i = 0; n > 0 && i < n / 3; i++) { + of_property_read_u32_index(node, "apple,retention-workaround", 3 * i, + &ps->retention[i].offset); + of_property_read_u32_index(node, "apple,retention-workaround", 3 * i + 1, + &ps->retention[i].mask); + of_property_read_u32_index(node, "apple,retention-workaround", 3 * i + 2, + &ps->retention[i].value); + if (ps->retention[i].value & ~ps->retention[i].mask) + return dev_err_probe(dev, -EINVAL, "bad retention workaround\n"); + } + ps->nr_retention = n > 0 ? n / 3 : 0; + } else if (of_property_present(node, "apple,retention-workaround")) { + return dev_err_probe(dev, -EINVAL, + "retention workaround needs a consumer switch\n"); + } ps->sleep_retained = of_property_read_bool(node, "apple,sleep-retained"); /* * Only T8132 hands over inherited-on domains with this contract. Older @@ -794,6 +838,245 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) return ret; } +static struct platform_driver apple_pmgr_ps_driver; + +/* Wait for the masked state word to read @want; @cur gets the last value. */ +static int apple_pmgr_ps_wait(struct apple_pmgr_ps *ps, u32 mask, u32 want, u32 *cur) +{ + return regmap_read_poll_timeout_atomic(ps->regmap, ps->offset, *cur, + (*cur & mask) == want, 1, + APPLE_PMGR_PS_CONSUMER_TIMEOUT); +} + +/* Write a retention register and read it back. */ +static int apple_pmgr_ps_retention_set(struct apple_pmgr_ps *ps, unsigned int i, u32 value) +{ + u32 now; + int ret; + + ret = regmap_write(ps->regmap, ps->retention[i].offset, value); + if (!ret) + ret = regmap_read(ps->regmap, ps->retention[i].offset, &now); + return ret ?: now == value ? 0 : -EIO; +} + +/* Fully on (target and actual), and not in reset, busy or disabled. */ +static bool apple_pmgr_ps_consumer_ready(u32 reg) +{ + return (reg & APPLE_PMGR_PS_LEVELS) == APPLE_PMGR_PS_ALL_ON && + !(reg & (APPLE_PMGR_RESET | APPLE_PMGR_PS_RESET | APPLE_PMGR_BUSY | + APPLE_PMGR_DEV_DISABLE)); +} + +/* + * The native power-off of a consumer-switched state: from fully on, disable + * automatic power management while staying on, set the retention workaround, + * then power off (clearing the history bits) and wait for it. + */ +static int apple_pmgr_ps_consumer_state_off(struct apple_pmgr_ps *ps) +{ + u32 reg, cur; + int ret; + + if (ps->consumer_failed) + return -EIO; + if (ps->consumer_off) + return -EALREADY; + ret = regmap_read(ps->regmap, ps->offset, ®); + if (ret) + goto fail; + if (!apple_pmgr_ps_consumer_ready(reg)) + return -EBUSY; /* Not touched: not a failure of this state. */ + for (unsigned int i = 0; i < ps->nr_retention; i++) { + ret = regmap_read(ps->regmap, ps->retention[i].offset, &ps->retention[i].saved); + if (ret) + goto fail; + } + ps->consumer_saved = reg; + reg &= ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS); + if (ps->consumer_saved & APPLE_PMGR_AUTO_ENABLE) { + ret = regmap_write(ps->regmap, ps->offset, reg); + if (!ret) + ret = apple_pmgr_ps_wait(ps, APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_PS_LEVELS, + APPLE_PMGR_PS_ALL_ON, &cur); + if (ret) + goto fail; + } + for (unsigned int i = 0; i < ps->nr_retention; i++) { + ret = apple_pmgr_ps_retention_set(ps, i, (ps->retention[i].saved & + ~ps->retention[i].mask) | + ps->retention[i].value); + if (ret) + goto fail; + } + ps->consumer_off = true; + reg = (reg & ~APPLE_PMGR_PS_TARGET) | APPLE_PMGR_FLAGS | + FIELD_PREP(APPLE_PMGR_PS_TARGET, APPLE_PMGR_PS_PWRGATE); + ret = regmap_write(ps->regmap, ps->offset, reg); + if (!ret) + ret = apple_pmgr_ps_wait(ps, APPLE_PMGR_PS_LEVELS | APPLE_PMGR_AUTO_ENABLE, 0, + &cur); + if (ret) + goto fail; + dev_dbg(ps->dev, "PS %s: consumer power-off: %#x -> %#x\n", ps->genpd.name, + ps->consumer_saved, cur); + return 0; +fail: + ps->consumer_failed = true; + dev_err(ps->dev, "PS %s: consumer power-off failed: %d\n", ps->genpd.name, ret); + return ret; +} + +/* + * The native power-on: power on with automatic power management disabled, + * give the retention registers' workaround bits back the values they had + * (only those: power-off changed no other), then automatic power management + * as it was. + */ +static int apple_pmgr_ps_consumer_state_on(struct apple_pmgr_ps *ps) +{ + u32 reg, cur, now; + int ret; + + if (ps->consumer_failed) + return -EIO; + if (!ps->consumer_off) + return -EALREADY; + reg = (ps->consumer_saved & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | + APPLE_PMGR_BUSY | APPLE_PMGR_PS_ACTUAL)) | + FIELD_PREP(APPLE_PMGR_PS_TARGET, APPLE_PMGR_PS_ACTIVE); + ret = regmap_write(ps->regmap, ps->offset, reg); + if (!ret) + ret = apple_pmgr_ps_wait(ps, APPLE_PMGR_PS_LEVELS, APPLE_PMGR_PS_ALL_ON, &cur); + if (ret) + goto fail; + for (unsigned int i = 0; i < ps->nr_retention; i++) { + u32 mask = ps->retention[i].mask; + + ret = regmap_read(ps->regmap, ps->retention[i].offset, &now); + if (!ret) + ret = apple_pmgr_ps_retention_set(ps, i, (now & ~mask) | + (ps->retention[i].saved & mask)); + if (ret) + goto fail; + } + if (ps->consumer_saved & APPLE_PMGR_AUTO_ENABLE) { + ret = regmap_write(ps->regmap, ps->offset, reg | APPLE_PMGR_AUTO_ENABLE); + if (!ret) + ret = apple_pmgr_ps_wait(ps, APPLE_PMGR_AUTO_ENABLE, APPLE_PMGR_AUTO_ENABLE, + &cur); + if (ret) + goto fail; + } + ps->consumer_off = false; + dev_dbg(ps->dev, "PS %s: consumer power-on: %#x\n", ps->genpd.name, cur); + return 0; +fail: + ps->consumer_failed = true; + dev_err(ps->dev, "PS %s: consumer power-on failed: %d\n", ps->genpd.name, ret); + return ret; +} + +/* The consumer-switched states from @consumer's domain up, child first. */ +static int apple_pmgr_ps_consumer_chain(struct device *consumer, + struct apple_pmgr_ps **chain) +{ + struct device_node *np = of_parse_phandle(consumer->of_node, "power-domains", 0); + int n = 0; + + while (np) { + struct platform_device *pdev = of_find_device_by_node(np); + struct apple_pmgr_ps *ps = NULL; + struct device_node *parent; + + if (pdev && pdev->dev.driver == &apple_pmgr_ps_driver.driver) + ps = platform_get_drvdata(pdev); + if (pdev) + put_device(&pdev->dev); + if (!ps || !ps->consumer_switched || n == APPLE_PMGR_PS_MAX_CHAIN) { + of_node_put(np); + break; + } + chain[n++] = ps; + parent = of_parse_phandle(np, "power-domains", 0); + of_node_put(np); + np = parent; + } + return n ?: -ENODEV; +} + +static int apple_pmgr_ps_consumer_switch(struct apple_pmgr_ps *ps, bool on) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&ps->genpd.slock, flags); + ret = on ? apple_pmgr_ps_consumer_state_on(ps) : apple_pmgr_ps_consumer_state_off(ps); + spin_unlock_irqrestore(&ps->genpd.slock, flags); + return ret; +} + +/* + * Whether apple_pmgr_ps_consumer_off() can run for @consumer, changing + * nothing: the whole chain consumer-switched, not failed and not off. Not + * whether each state is fully on: under automatic power management the + * J713 DCP CPU state reads level 0 while its CPU idles, and is fully on + * again once the CPU has been stopped, which the power-off then requires. + */ +int apple_pmgr_ps_consumer_check(struct device *consumer) +{ + struct apple_pmgr_ps *chain[APPLE_PMGR_PS_MAX_CHAIN]; + int n = apple_pmgr_ps_consumer_chain(consumer, chain), ret = 0; + + if (n < 0) + return n; + for (int i = 0; i < n && !ret; i++) { + struct apple_pmgr_ps *ps = chain[i]; + unsigned long flags; + + spin_lock_irqsave(&ps->genpd.slock, flags); + if (ps->consumer_failed) + ret = -EIO; + else if (ps->consumer_off) + ret = -EALREADY; + spin_unlock_irqrestore(&ps->genpd.slock, flags); + } + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmgr_ps_consumer_check); + +int apple_pmgr_ps_consumer_off(struct device *consumer) +{ + struct apple_pmgr_ps *chain[APPLE_PMGR_PS_MAX_CHAIN]; + int n = apple_pmgr_ps_consumer_chain(consumer, chain), ret; + + if (n < 0) + return n; + for (int i = 0; i < n; i++) { + ret = apple_pmgr_ps_consumer_switch(chain[i], false); + if (ret) + return ret; + } + return 0; +} +EXPORT_SYMBOL_GPL(apple_pmgr_ps_consumer_off); + +int apple_pmgr_ps_consumer_on(struct device *consumer) +{ + struct apple_pmgr_ps *chain[APPLE_PMGR_PS_MAX_CHAIN]; + int n = apple_pmgr_ps_consumer_chain(consumer, chain), ret; + + if (n < 0) + return n; + for (int i = n - 1; i >= 0; i--) { + ret = apple_pmgr_ps_consumer_switch(chain[i], true); + if (ret) + return ret; + } + return 0; +} +EXPORT_SYMBOL_GPL(apple_pmgr_ps_consumer_on); + static const struct of_device_id apple_pmgr_ps_of_match[] = { { .compatible = "apple,t8103-pmgr-pwrstate" }, { .compatible = "apple,pmgr-pwrstate" }, diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index c13c03f9d92ab..21ae3a51f9966 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -957,25 +957,10 @@ static void apple_mbox_v8_rx_stop(struct apple_mbox *mbox) cancel_delayed_work_sync(&priv->rx_poll_work); } -int apple_mbox_start(struct apple_mbox *mbox) +/* The mailbox-level enables, which the mailbox's power domain does not keep. */ +static void apple_mbox_enable_hw(struct apple_mbox *mbox) { u32 control; - int ret = 0; - - mutex_lock(&mbox->lifecycle_lock); - if (mbox->removing) { - ret = -ENODEV; - goto out_unlock; - } - if (mbox->active) - goto out_unlock; - - ret = pm_runtime_resume_and_get(mbox->dev); - if (ret) { - dev_err(mbox->dev, "mailbox start: runtime resume failed: %d\n", ret); - apple_mbox_clear_resume_error(mbox); - goto out_unlock; - } if (mbox->hw->ap_initializes_mailboxes) { /* The primary T8140 outbox requires its enable bit before use. */ @@ -996,6 +981,28 @@ int apple_mbox_start(struct apple_mbox *mbox) mbox->hw->irq_bit_send_empty, mbox->regs + mbox->hw->irq_enable); } +} + +int apple_mbox_start(struct apple_mbox *mbox) +{ + int ret = 0; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing) { + ret = -ENODEV; + goto out_unlock; + } + if (mbox->active) + goto out_unlock; + + ret = pm_runtime_resume_and_get(mbox->dev); + if (ret) { + dev_err(mbox->dev, "mailbox start: runtime resume failed: %d\n", ret); + apple_mbox_clear_resume_error(mbox); + goto out_unlock; + } + + apple_mbox_enable_hw(mbox); if (apple_mbox_has_rx_poller(mbox)) { WRITE_ONCE(mbox->active, true); @@ -1036,6 +1043,25 @@ void apple_mbox_stop(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_stop); +/* + * The mailbox's power domain was off while it stayed started (its consumer + * powered its co-processor down and up again around it): set the + * mailbox-level enables again before the co-processor runs. + */ +int apple_mbox_restore(struct apple_mbox *mbox) +{ + int ret = 0; + + mutex_lock(&mbox->lifecycle_lock); + if (mbox->removing || !mbox->active) + ret = -ENODEV; + else + apple_mbox_enable_hw(mbox); + mutex_unlock(&mbox->lifecycle_lock); + return ret; +} +EXPORT_SYMBOL(apple_mbox_restore); + void apple_mbox_hold_rx(struct apple_mbox *mbox) { disable_irq(mbox->irq_recv_not_empty); diff --git a/drivers/soc/apple/pmp-display.c b/drivers/soc/apple/pmp-display.c index fed0b13bdfe83..b32e155ec09f0 100644 --- a/drivers/soc/apple/pmp-display.c +++ b/drivers/soc/apple/pmp-display.c @@ -24,6 +24,8 @@ /* AGX: command 0x0f, logical device 5, state 1. The reply carries no tag. */ #define PMP_AGX_REQUEST 0x0020f50100000000ULL #define PMP_DISP_REQUEST 0x0020f80100000000ULL +/* DISP, state 0: released while its DCP has powered the display domains down. */ +#define PMP_DISP_RELEASE 0x0020f80000000000ULL #define PMP_POWER_REPLY 0x0021000000000000ULL #define PMP_ENDPOINT 0x20 @@ -41,6 +43,8 @@ struct pmp_power_vote { /* A failed PMP owner invalidates every accepted vote as well. */ int provider_error; enum pmp_power_kind pending; + /* The pending request releases its vote rather than taking it. */ + bool releasing; bool held[PMP_POWER_COUNT]; }; @@ -48,7 +52,11 @@ static void pmp_vote_fail(struct pmp_power_vote *s, int error) { if (!s->error) s->error = error < 0 ? error : -EIO; + /* A release with an unknown outcome no longer counts as held. */ + if (s->pending && s->releasing) + s->held[s->pending] = false; s->pending = PMP_POWER_NONE; + s->releasing = false; } /* The Rust owner latches every ambiguous transaction as a provider failure. @@ -57,10 +65,12 @@ static void pmp_vote_fail(struct pmp_power_vote *s, int error) */ static void pmp_vote_send_failed(struct pmp_power_vote *s) { - if (s->provider_error) + if (s->provider_error) { pmp_vote_fail(s, s->provider_error); - else + } else { s->pending = PMP_POWER_NONE; + s->releasing = false; + } } /* One means send the request, zero means the same retained vote is held. */ @@ -80,6 +90,52 @@ static int pmp_vote_begin(struct pmp_power_vote *s, enum pmp_power_kind kind) return 1; } +/* One means send the release, zero means the vote is not held. */ +static int pmp_vote_begin_release(struct pmp_power_vote *s, enum pmp_power_kind kind) +{ + if (kind <= PMP_POWER_NONE || kind >= PMP_POWER_COUNT) + return -EINVAL; + if (s->provider_error) + return s->provider_error; + if (!s->held[kind]) + return 0; + if (s->error) + return s->error; + if (s->pending) + return -EBUSY; + s->pending = kind; + s->releasing = true; + return 1; +} + +/* + * Whether a vote can be released now and taken again later, changing + * nothing: held, no request in flight, and admission open (a failed untagged + * request closes it, and the later get is a new request). + */ +static int pmp_vote_check_release(struct pmp_power_vote *s, enum pmp_power_kind kind) +{ + if (kind <= PMP_POWER_NONE || kind >= PMP_POWER_COUNT) + return -EINVAL; + if (s->provider_error) + return s->provider_error; + if (!s->held[kind]) + return -ENODEV; + if (s->error) + return s->error; + return s->pending ? -EBUSY : 0; +} + +/* Released, or the outcome is unknown and admission is closed. */ +static int pmp_vote_finish_release(struct pmp_power_vote *s, enum pmp_power_kind kind) +{ + if (s->provider_error) + return s->provider_error; + if (s->pending == kind && s->releasing) + pmp_vote_fail(s, -ETIMEDOUT); + return s->held[kind] || s->error ? s->error ?: -EIO : 0; +} + /* Accepted votes are retained even if a different untagged request fails. */ static int pmp_vote_check(struct pmp_power_vote *s, enum pmp_power_kind kind) { @@ -110,8 +166,9 @@ static void pmp_vote_reply(struct pmp_power_vote *s, u8 endpoint, u64 word) pmp_vote_fail(s, -EPROTO); return; } - s->held[s->pending] = true; + s->held[s->pending] = !s->releasing; s->pending = PMP_POWER_NONE; + s->releasing = false; } static inline bool j713_disp_table(const void *data, size_t size) @@ -338,7 +395,7 @@ static int pmp_power_prepare(struct device *consumer, enum pmp_power_kind kind) return ret; } -static int pmp_power_check(struct device *consumer, enum pmp_power_kind kind) +static int pmp_power_check(struct device *consumer, enum pmp_power_kind kind, bool release) { struct apple_pmp_display *p = pmp_power_consumer(consumer, kind); unsigned long flags; @@ -347,7 +404,7 @@ static int pmp_power_check(struct device *consumer, enum pmp_power_kind kind) if (IS_ERR(p)) return PTR_ERR(p); spin_lock_irqsave(&p->lock, flags); - ret = pmp_vote_check(&p->vote, kind); + ret = release ? pmp_vote_check_release(&p->vote, kind) : pmp_vote_check(&p->vote, kind); spin_unlock_irqrestore(&p->lock, flags); return ret; } @@ -405,6 +462,55 @@ static int pmp_power_get(struct device *consumer, enum pmp_power_kind kind) return ret; } +/* + * Release the DISP vote while the consumer has powered the display domains + * down; apple_pmp_j713_display_get() takes it again before power-up. An + * ambiguous release closes admission: the vote is then neither held nor + * taken again. + */ +static int pmp_power_put(struct device *consumer, enum pmp_power_kind kind) +{ + struct apple_pmp_display *p = pmp_power_consumer(consumer, kind); + unsigned long flags; + int ret; + + if (IS_ERR(p)) + return PTR_ERR(p); + if (kind != PMP_POWER_DISP) + return -EINVAL; + device_lock(p->dev); + mutex_lock(&p->request); + if (p->dev->driver != p->driver || !dev_get_drvdata(p->dev)) { + ret = -ENODEV; + apple_pmp_display_fail(p, ret); + goto out; + } + spin_lock_irqsave(&p->lock, flags); + reinit_completion(&p->reply); + ret = pmp_vote_begin_release(&p->vote, kind); + spin_unlock_irqrestore(&p->lock, flags); + if (ret <= 0) + goto out; + ret = p->send(p->cookie, PMP_DISP_RELEASE); + if (ret) { + spin_lock_irqsave(&p->lock, flags); + pmp_vote_send_failed(&p->vote); + spin_unlock_irqrestore(&p->lock, flags); + goto out; + } + spin_lock_irqsave(&p->lock, flags); + pmp_vote_reply(&p->vote, PMP_ENDPOINT, PMP_POWER_REPLY); + ret = pmp_vote_finish_release(&p->vote, kind); + spin_unlock_irqrestore(&p->lock, flags); + if (!ret) + dev_dbg(p->dev, "Retained DISP vote released: request=%016llx\n", + PMP_DISP_RELEASE); +out: + mutex_unlock(&p->request); + device_unlock(p->dev); + return ret; +} + /* Probe creates the retained supplier links; get/check never change owners. */ int apple_pmp_j713_display_prepare(struct device *consumer) { @@ -420,13 +526,19 @@ EXPORT_SYMBOL_GPL(apple_pmp_j713_gpu_prepare); int apple_pmp_j713_display_check(struct device *consumer) { - return apple_pmp_j713_board() ? pmp_power_check(consumer, PMP_POWER_DISP) : -ENODEV; + return apple_pmp_j713_board() ? pmp_power_check(consumer, PMP_POWER_DISP, false) : -ENODEV; } EXPORT_SYMBOL_GPL(apple_pmp_j713_display_check); +int apple_pmp_j713_display_check_release(struct device *consumer) +{ + return apple_pmp_j713_board() ? pmp_power_check(consumer, PMP_POWER_DISP, true) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_pmp_j713_display_check_release); + int apple_pmp_j713_gpu_check(struct device *consumer) { - return apple_pmp_j713_board() ? pmp_power_check(consumer, PMP_POWER_AGX) : -ENODEV; + return apple_pmp_j713_board() ? pmp_power_check(consumer, PMP_POWER_AGX, false) : -ENODEV; } EXPORT_SYMBOL_GPL(apple_pmp_j713_gpu_check); @@ -436,6 +548,12 @@ int apple_pmp_j713_display_get(struct device *consumer) } EXPORT_SYMBOL_GPL(apple_pmp_j713_display_get); +int apple_pmp_j713_display_put(struct device *consumer) +{ + return apple_pmp_j713_board() ? pmp_power_put(consumer, PMP_POWER_DISP) : -ENODEV; +} +EXPORT_SYMBOL_GPL(apple_pmp_j713_display_put); + int apple_pmp_j713_gpu_get(struct device *consumer) { return apple_pmp_j713_board() ? pmp_power_get(consumer, PMP_POWER_AGX) : -ENODEV; diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 2212509efd82c..39cc8821e0ba4 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -487,6 +487,8 @@ unsafe extern "C" fn display_send(cookie: *const kernel::ffi::c_void, word: u64) match word { 0x0020f50100000000 => unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 1) }, 0x0020f80100000000 => unsafe { bindings::apple_pmp_set_device_power(0x0f, 8, 1) }, + // DISP released while its DCP has powered the display domains down. + 0x0020f80000000000 => unsafe { bindings::apple_pmp_set_device_power(0x0f, 8, 0) }, _ => EINVAL.to_errno(), } } diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 2ad4d96bec768..577f68fad529c 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -65,6 +65,15 @@ struct apple_rtkit { struct work_struct sync_ack_work; DECLARE_BITMAP(endpoints, APPLE_RTKIT_MAX_ENDPOINTS); + /* Endpoints started in this session; kept by a retained wake. */ + DECLARE_BITMAP(started, APPLE_RTKIT_MAX_ENDPOINTS); + /* + * apple_rtkit_wake_retained() in progress: the endpoint map is answered + * with the started endpoints and nothing is started again; a QUIESCED + * report gets one ON request. + */ + bool retained_wake; + bool retained_on_sent; struct apple_rtkit_shmem ioreport_buffer; struct apple_rtkit_shmem coreanalytics_buffer; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 3b821415d54d8..b6b27a713442a 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -45,6 +45,8 @@ enum { APPLE_RTKIT_MGMT_EPMAP_REPLY = 8, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb, + /* Sent by some co-processors once idle after an IOP IDLE request. */ + APPLE_RTKIT_MGMT_IDLE_NOTIFY = 0xc, }; #define APPLE_RTKIT_MGMT_HELLO_MINVER GENMASK_ULL(15, 0) @@ -229,6 +231,55 @@ static void apple_rtkit_management_rx_hello(struct apple_rtkit *rtk, u64 msg) complete_all(&rtk->epmap_completion); } +/* + * Endpoint map of a retained wake: the endpoints and their buffers are still + * set up, so answer with the started endpoints of this part of the map, as + * the native retained wake does, and start nothing. A started endpoint that + * the firmware no longer offers fails the wake. + */ +static void apple_rtkit_management_rx_epmap_retained(struct apple_rtkit *rtk, u64 msg, + u32 base, unsigned long bitmap) +{ + unsigned long started = 0; + u64 reply; + int i; + + for (i = 0; i < 32 && 32 * base + i < APPLE_RTKIT_MAX_ENDPOINTS; i++) + if (test_bit(32 * base + i, rtk->started)) + started |= BIT(i); + if (started & ~bitmap) { + dev_err(rtk->dev, "RTKit: retained endpoints %#lx missing from map %#lx (base %u)\n", + started & ~bitmap, bitmap, base); + rtk->boot_result = -EPROTO; + complete_all(&rtk->epmap_completion); + return; + } + /* + * Bit 0 of a reply is also MORE: a last part cannot echo endpoint + * 32 * base there. The DCP's map has two parts, and its endpoint 0 + * (management) is in the first. + */ + if ((msg & APPLE_RTKIT_MGMT_EPMAP_LAST) && (started & APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE)) { + dev_err(rtk->dev, "RTKit: retained endpoint %u cannot be echoed in the last map part\n", + 32 * base); + rtk->boot_result = -EPROTO; + complete_all(&rtk->epmap_completion); + return; + } + + reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base) | started; + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) + reply |= APPLE_RTKIT_MGMT_EPMAP_LAST; + else + reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE; + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply); + + if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) { + rtk->boot_result = 0; + complete_all(&rtk->epmap_completion); + } +} + static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) { int i, ep, ret; @@ -261,6 +312,11 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) } } + if (rtk->retained_wake) { + apple_rtkit_management_rx_epmap_retained(rtk, msg, base, bitmap); + return; + } + reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base); if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST) reply |= APPLE_RTKIT_MGMT_EPMAP_LAST; @@ -330,6 +386,28 @@ static void apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit *rtk, rtk->iop_power_state, new_state); rtk->iop_power_state = new_state; + /* + * A retained wake: a co-processor may report QUIESCED when it starts + * running and wait for an ON request before its HELLO. Ask once, and + * wait for ON. + */ + if (rtk->retained_wake && new_state != APPLE_RTKIT_PWR_STATE_ON) { + if (new_state == APPLE_RTKIT_PWR_STATE_QUIESCED && !rtk->retained_on_sent) { + rtk->retained_on_sent = true; + apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, + FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, + APPLE_RTKIT_PWR_STATE_ON)); + return; + } + dev_err(rtk->dev, "RTKit: unexpected IOP power state 0x%x on retained wake\n", + new_state); + /* Whichever wait the wake is in: the map, or IOP ON after it. */ + rtk->boot_result = -EPROTO; + complete_all(&rtk->epmap_completion); + complete_all(&rtk->iop_pwr_ack_completion); + return; + } + complete_all(&rtk->iop_pwr_ack_completion); } @@ -362,6 +440,9 @@ static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK: apple_rtkit_management_rx_ap_pwr_ack(rtk, msg); break; + case APPLE_RTKIT_MGMT_IDLE_NOTIFY: + dev_dbg(rtk->dev, "RTKit: co-processor idle\n"); + break; default: dev_warn( rtk->dev, @@ -1245,6 +1326,8 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) endpoint); ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg); } + if (!ret) + set_bit(endpoint, rtk->started); return ret; } EXPORT_SYMBOL_GPL(apple_rtkit_start_ep); @@ -1423,6 +1506,7 @@ static struct apple_rtkit *__apple_rtkit_init(struct device *dev, void *cookie, bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); + set_bit(APPLE_RTKIT_EP_MGMT, rtk->started); if (mbox_name) rtk->mbox = apple_mbox_get_byname(dev, mbox_name); @@ -1487,6 +1571,18 @@ EXPORT_SYMBOL_GPL(apple_rtkit_init_adopted); static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, struct completion *c); +/* + * A retained wake is not interrupted by a signal to its caller: it has + * already started the co-processor's CPU, and nothing would undo that. + */ +static long apple_rtkit_wait_timeout(struct apple_rtkit *rtk, struct completion *c, + unsigned long timeout) +{ + if (READ_ONCE(rtk->retained_wake)) + return wait_for_completion_timeout(c, timeout); + return wait_for_completion_interruptible_timeout(c, timeout); +} + /* * apple,sync-syslog-ack (AsahiLinux/linux#607): the mailbox can leave * messages queued without another interrupt during bootstrap, so poll it @@ -1507,8 +1603,7 @@ static int apple_rtkit_wait(struct apple_rtkit *rtk, struct completion *c) if (ret < 0) return ret; - t = wait_for_completion_interruptible_timeout(c, - msecs_to_jiffies(10)); + t = apple_rtkit_wait_timeout(rtk, c, msecs_to_jiffies(10)); if (t < 0) return t; if (t > 0) @@ -1535,7 +1630,7 @@ static int apple_rtkit_wait_for_completion(struct apple_rtkit *rtk, unsigned long timeout = msecs_to_jiffies(rtk->handshake_timeout_ms); long t; - t = wait_for_completion_interruptible_timeout(c, timeout); + t = apple_rtkit_wait_timeout(rtk, c, timeout); if (t < 0) return t; else if (t == 0) @@ -1580,6 +1675,8 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); + bitmap_zero(rtk->started, APPLE_RTKIT_MAX_ENDPOINTS); + set_bit(APPLE_RTKIT_EP_MGMT, rtk->started); reinit_completion(&rtk->epmap_completion); reinit_completion(&rtk->coreanalytics_completion); @@ -1697,6 +1794,9 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) dev_err(rtk->dev, "RTKit: IOP power acknowledgment failed: %d\n", ret); return ret; } + /* A retained wake fails on any IOP state but ON (still 0 otherwise). */ + if (rtk->boot_result) + return rtk->boot_result; return early_ap_power ? 0 : apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); @@ -1831,6 +1931,47 @@ int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_wake_early_ap); +/* + * Wake a co-processor whose CPU was stopped after apple_rtkit_idle(), with + * its firmware, endpoints and shared buffers kept (and its power domain + * possibly off meanwhile). The caller holds RX since the idle; this restores + * the mailbox, resumes RX and calls @start, which starts the CPU. The + * firmware then says HELLO (possibly after reporting QUIESCED, which gets an + * ON request) and sends its endpoint map, answered with the started + * endpoints; no INIT is sent and nothing is started or allocated again. + * Then IOP ON is awaited and AP ON requested, as for any boot. + */ +int apple_rtkit_wake_retained(struct apple_rtkit *rtk, int (*start)(void *), void *ctx) +{ + int ret; + + if (apple_rtkit_is_running(rtk) || rtk->crashed || rtk->shutting_down || + rtk->iop_power_state != APPLE_RTKIT_PWR_STATE_IDLE || !start) + return -EINVAL; + + reinit_completion(&rtk->epmap_completion); + reinit_completion(&rtk->iop_pwr_ack_completion); + reinit_completion(&rtk->ap_pwr_ack_completion); + rtk->boot_result = -EINPROGRESS; + rtk->retained_on_sent = false; + WRITE_ONCE(rtk->retained_wake, true); + + ret = apple_mbox_restore(rtk->mbox); + if (ret) + goto out; + apple_rtkit_resume_rx(rtk); + ret = start(ctx); + if (ret) + goto out; + ret = apple_rtkit_boot_common(rtk, false); +out: + WRITE_ONCE(rtk->retained_wake, false); + if (ret) + dev_err(rtk->dev, "RTKit: retained wake failed: %d\n", ret); + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_wake_retained); + void apple_rtkit_free(struct apple_rtkit *rtk) { apple_rtkit_stop_rx(rtk); diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index ce81d6ec5bcd3..10d05a7cc042b 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -38,6 +38,15 @@ int apple_dart_dcp_dispext_stream_ok(struct device *piodma, unsigned int sid); void *apple_dart_dcp_dispext_scanout_alloc(struct device *piodma, size_t size, u64 *dva); /* An adopted external display DART's state after its power-up; reads only. */ int apple_dart_dcp_dispext_state(struct device_node *consumer); +/* + * Internal display DART of a DCP consumer (the DCP or its PIODMA), around a + * power-down of the display domains by the DCP: streams off, flushed and + * clock protection cleared before; state checked and streams back after. + * The check says, changing nothing, whether the power-off can run now. + */ +int apple_dart_dcp_power_check(struct device *dev); +int apple_dart_dcp_power_off(struct device *dev); +int apple_dart_dcp_power_on(struct device *dev); /* * Remappable host windows in SID0 of an external display DART: one @@ -56,6 +65,9 @@ void *apple_dart_dcp_scanout_alloc(struct device *dev, size_t size, dma_addr_t * #else static inline int apple_dart_dcp_check(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_dcp_power_check(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_dcp_power_off(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_dcp_power_on(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_dcp_check_buffer(struct device *dev, dma_addr_t iova, size_t size) { return -EOPNOTSUPP; diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index 7a07772cc5967..d7fc9dd4134f1 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -100,6 +100,7 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name); int apple_mbox_start(struct apple_mbox *mbox); void apple_mbox_stop(struct apple_mbox *mbox); +int apple_mbox_restore(struct apple_mbox *mbox); int apple_mbox_poll(struct apple_mbox *mbox); /* * Leave received messages in the hardware FIFO, without an interrupt, until diff --git a/include/linux/soc/apple/pmc.h b/include/linux/soc/apple/pmc.h index be391b83d9752..da0635015f52d 100644 --- a/include/linux/soc/apple/pmc.h +++ b/include/linux/soc/apple/pmc.h @@ -26,7 +26,30 @@ bool apple_pmc_voter_enabled(struct device *pmc, unsigned int agent, u32 apple_pmc_floor_read(struct device *pmc, unsigned int agent); int apple_pmc_floor_update(struct device *pmc, unsigned int agent, u32 mask, u32 value); +/* + * Clear, and later restore, the display pipe interface enables named by + * @consumer's apple,pmc-interface (an apple,consumer-switched interface). + * The check says, changing nothing, whether they can be cleared now. + */ +int apple_pmc_interface_consumer_check(struct device *consumer); +int apple_pmc_interface_consumer_off(struct device *consumer); +int apple_pmc_interface_consumer_on(struct device *consumer); #else +static inline int apple_pmc_interface_consumer_check(struct device *consumer) +{ + return -EOPNOTSUPP; +} + +static inline int apple_pmc_interface_consumer_off(struct device *consumer) +{ + return -EOPNOTSUPP; +} + +static inline int apple_pmc_interface_consumer_on(struct device *consumer) +{ + return -EOPNOTSUPP; +} + static inline unsigned int apple_pmc_rail_count(struct device *pmc) { return 0; diff --git a/include/linux/soc/apple/pmgr-pwrstate.h b/include/linux/soc/apple/pmgr-pwrstate.h index 63311cfd7d979..e09a7ee3e031c 100644 --- a/include/linux/soc/apple/pmgr-pwrstate.h +++ b/include/linux/soc/apple/pmgr-pwrstate.h @@ -4,12 +4,26 @@ #include +struct device; struct device_node; #if IS_ENABLED(CONFIG_APPLE_PMGR_PWRSTATE) int apple_pmgr_ps_cycle(struct device_node *np); +/* + * Power off (child first) or on (parent first) the chain of + * apple,consumer-switched power states that starts at @consumer's power + * domain. The consumer has stopped its coprocessor before power-off and + * starts it again after power-on. The check says, changing nothing, whether + * the power-off can run now. + */ +int apple_pmgr_ps_consumer_check(struct device *consumer); +int apple_pmgr_ps_consumer_off(struct device *consumer); +int apple_pmgr_ps_consumer_on(struct device *consumer); #else static inline int apple_pmgr_ps_cycle(struct device_node *np) { return -EOPNOTSUPP; } +static inline int apple_pmgr_ps_consumer_check(struct device *consumer) { return -EOPNOTSUPP; } +static inline int apple_pmgr_ps_consumer_off(struct device *consumer) { return -EOPNOTSUPP; } +static inline int apple_pmgr_ps_consumer_on(struct device *consumer) { return -EOPNOTSUPP; } #endif #endif diff --git a/include/linux/soc/apple/pmp-display.h b/include/linux/soc/apple/pmp-display.h index 7db16206d4e8a..e65effb337aa8 100644 --- a/include/linux/soc/apple/pmp-display.h +++ b/include/linux/soc/apple/pmp-display.h @@ -31,12 +31,17 @@ void apple_pmp_display_fail(struct apple_pmp_display *display, int error); struct apple_pmp_display *apple_pmp_display_bound_owner(struct device *dev); /* J713 votes share the existing PMP owner's untagged request stream. - * Prepare is probe-only; accepted DISP/AGX votes and supplier links persist - * until reboot. + * Prepare is probe-only; supplier links persist until reboot, and so do + * accepted votes, except that the DCP releases its DISP vote while it has + * powered the display domains down (put) and takes it again (get). Before + * that it asks, changing nothing, whether the vote can be released now and + * taken again later (check_release). */ int apple_pmp_j713_display_prepare(struct device *consumer); int apple_pmp_j713_display_get(struct device *consumer); +int apple_pmp_j713_display_put(struct device *consumer); int apple_pmp_j713_display_check(struct device *consumer); +int apple_pmp_j713_display_check_release(struct device *consumer); int apple_pmp_j713_gpu_prepare(struct device *consumer); int apple_pmp_j713_gpu_get(struct device *consumer); int apple_pmp_j713_gpu_check(struct device *consumer); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 86cdb8426488d..ac646240f8587 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -234,6 +234,14 @@ int apple_rtkit_wake(struct apple_rtkit *rtk); /* Wake helpers which require AP readiness before their IOP power acknowledgment. */ int apple_rtkit_wake_early_ap(struct apple_rtkit *rtk); +/* + * Wake a co-processor stopped after apple_rtkit_idle() with its firmware, + * endpoints and buffers kept: restore the mailbox, resume RX, call @start to + * start its CPU, then complete HELLO, the endpoint map (with the started + * endpoints, no INIT and no new endpoint start) and IOP/AP ON. + */ +int apple_rtkit_wake_retained(struct apple_rtkit *rtk, int (*start)(void *), void *ctx); + /* * Shutdown the co-processor */ diff --git a/tools/asahi/rtkit-oslog-tests.py b/tools/asahi/rtkit-oslog-tests.py index 160c163758d54..87c3654fb8328 100644 --- a/tools/asahi/rtkit-oslog-tests.py +++ b/tools/asahi/rtkit-oslog-tests.py @@ -49,7 +49,7 @@ def function(source, name): int sync_ack_fifo; bool sync_syslog_ack, oslog_inherited, syslog_inherited, crashlog_inherited, crashed, shutting_down; unsigned iop_power_state, ap_power_state; - unsigned long endpoints[4]; + unsigned long endpoints[4], started[4]; char *syslog_msg_buffer; size_t syslog_n_entries, syslog_msg_size; int epmap_completion, iop_pwr_ack_completion, ap_pwr_ack_completion, coreanalytics_completion; diff --git a/tools/asahi/rtkit-retained-tests.py b/tools/asahi/rtkit-retained-tests.py new file mode 100755 index 0000000000000..b79a9cd542bd2 --- /dev/null +++ b/tools/asahi/rtkit-retained-tests.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Check the retained-wake endpoint map reply and IOP power acknowledgment.""" +import argparse +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +DRIVER = 'drivers/soc/apple/rtkit.c' + + +def function(source, name): + match = re.search(r'^(?:static )?(?:void|int|long) ' + name + r'\(', source, re.M) + assert match, name + opening = source.index('{', match.end()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + '\n' + + +def constants(source): + out = [] + for line in source.splitlines(): + if re.match(r'#define APPLE_RTKIT_MGMT_(TYPE|EPMAP_\w+|PWR_STATE)\b', line): + out.append(line) + enum = re.search(r'enum \{\n\tAPPLE_RTKIT_MGMT_HELLO = 1,.*?\n\};', source, re.S) + states = re.search(r'enum \{\n\tAPPLE_RTKIT_PWR_STATE_OFF.*?\n\};', source, re.S) + assert enum and states + return '\n'.join(out) + '\n' + enum.group(0) + '\n' + states.group(0) + '\n' + + +PREFIX = r''' +#include +#include +#include +#include +#include +typedef uint8_t u8; typedef uint32_t u32; typedef uint64_t u64; +#define BIT(n) (1UL << (n)) +#define BIT_ULL(n) (1ULL << (n)) +#define GENMASK_ULL(h,l) ((~0ULL << (l)) & (~0ULL >> (63-(h)))) +#define FIELD_GET(m,v) (((v) & (m)) >> __builtin_ctzll(m)) +#define FIELD_PREP(m,v) (((u64)(v) << __builtin_ctzll(m)) & (m)) +#define APPLE_RTKIT_MAX_ENDPOINTS 256 +#define dev_err(...) ((void)0) +#define dev_dbg(...) ((void)0) +struct apple_rtkit { + void *dev; + unsigned long started[APPLE_RTKIT_MAX_ENDPOINTS / 64]; + bool retained_wake, retained_on_sent; + int boot_result; + unsigned int iop_power_state; + int epmap_completion, iop_pwr_ack_completion; +}; +static bool test_bit(unsigned n, const unsigned long *b) { return b[n / 64] >> (n % 64) & 1; } +static void set_bit(unsigned n, unsigned long *b) { b[n / 64] |= 1UL << (n % 64); } +static void complete_all(int *c) { (*c)++; } +static unsigned sent; +static u8 sent_type; +static u64 sent_data; +static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type, u64 data) +{ + (void)rtk; sent++; sent_type = type; sent_data = data; return 0; +} +/* A signal is pending for every interruptible wait. */ +#define ERESTARTSYS 512 +#define READ_ONCE(x) (x) +struct completion { int unused; }; +static unsigned waited_uninterruptible, waited_interruptible; +static unsigned long wait_for_completion_timeout(struct completion *c, unsigned long t) +{ + (void)c; waited_uninterruptible++; return t; +} +static long wait_for_completion_interruptible_timeout(struct completion *c, unsigned long t) +{ + (void)c; (void)t; waited_interruptible++; return -ERESTARTSYS; +} +''' + +MAIN = r''' +static struct apple_rtkit fresh(void) +{ + struct apple_rtkit r = { .retained_wake = true, .boot_result = -EINPROGRESS }; + + /* Management, syslog, crashlog, debug, ioreport, oslog, tracekit, DCP 0x37. */ + for (unsigned ep = 0; ep < 11; ep++) + if ((0x51dUL >> ep) & 1) + set_bit(ep, r.started); + set_bit(0x37, r.started); + sent = 0; + return r; +} + +int main(void) +{ + struct apple_rtkit r = fresh(); + u64 map0 = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, 0) | 0x51d; + u64 map1 = APPLE_RTKIT_MGMT_EPMAP_LAST | FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, 1) | + 0x00867f3f; + + /* Each part of the map: exactly the started endpoints, MORE until the last. */ + apple_rtkit_management_rx_epmap_retained(&r, map0, 0, 0x51d); + assert(sent == 1 && sent_type == APPLE_RTKIT_MGMT_EPMAP_REPLY); + assert(sent_data == (0x51d | APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE)); + assert(r.epmap_completion == 0 && r.boot_result == -EINPROGRESS); + apple_rtkit_management_rx_epmap_retained(&r, map1, 1, 0x00867f3f); + assert(sent == 2 && sent_data == (APPLE_RTKIT_MGMT_EPMAP_LAST | + FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, 1) | BIT(0x37 - 32))); + assert(r.epmap_completion == 1 && r.boot_result == 0); + + /* A started endpoint missing from the map fails the wake; nothing is sent. */ + r = fresh(); + apple_rtkit_management_rx_epmap_retained(&r, map1 & ~(u64)BIT(0x37 - 32), 1, + 0x00867f3f & ~BIT(0x37 - 32)); + assert(sent == 0 && r.boot_result == -EPROTO && r.epmap_completion == 1); + + /* QUIESCED gets one ON request, then ON completes. */ + r = fresh(); + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_QUIESCED); + assert(sent == 1 && sent_type == APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE); + assert(sent_data == APPLE_RTKIT_PWR_STATE_ON && r.iop_pwr_ack_completion == 0); + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_ON); + assert(sent == 1 && r.iop_pwr_ack_completion == 1); + assert(r.iop_power_state == APPLE_RTKIT_PWR_STATE_ON); + + /* A last part cannot echo endpoint 32 * base: bit 0 is also MORE. */ + r = fresh(); + set_bit(32, r.started); + apple_rtkit_management_rx_epmap_retained(&r, map1, 1, 0x00867f3f); + assert(sent == 0 && r.boot_result == -EPROTO && r.epmap_completion == 1); + + /* + * A second QUIESCED, or any other state, fails the wake, before the map + * or while the wake waits for IOP ON after it. + */ + r = fresh(); + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_QUIESCED); + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_QUIESCED); + assert(sent == 1 && r.boot_result == -EPROTO); + assert(r.epmap_completion == 1 && r.iop_pwr_ack_completion == 1); + r = fresh(); + apple_rtkit_management_rx_epmap_retained(&r, map0, 0, 0x51d); + apple_rtkit_management_rx_epmap_retained(&r, map1, 1, 0x00867f3f); + assert(r.epmap_completion == 1 && r.boot_result == 0); + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_IDLE); + assert(r.boot_result == -EPROTO && r.iop_pwr_ack_completion == 1); + + /* Outside a retained wake every acknowledgment completes, as before. */ + r = fresh(); + r.retained_wake = false; + apple_rtkit_management_rx_iop_pwr_ack(&r, APPLE_RTKIT_PWR_STATE_QUIESCED); + assert(sent == 0 && r.iop_pwr_ack_completion == 1); + assert(r.iop_power_state == APPLE_RTKIT_PWR_STATE_QUIESCED); + + /* A retained wake waits through a signal; other waits stay interruptible. */ + struct completion c; + r = fresh(); + assert(apple_rtkit_wait_timeout(&r, &c, 7) == 7); + assert(waited_uninterruptible == 1 && waited_interruptible == 0); + r.retained_wake = false; + assert(apple_rtkit_wait_timeout(&r, &c, 7) == -ERESTARTSYS); + assert(waited_uninterruptible == 1 && waited_interruptible == 1); + + puts("PASS retained endpoint map replies, IOP acknowledgments and waits"); + return 0; +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--out', type=Path, help='keep build and result files here') + parser.add_argument('--cc', default='cc') + args = parser.parse_args() + source = (ROOT / DRIVER).read_text() + code = PREFIX + constants(source) + code += function(source, 'apple_rtkit_management_rx_epmap_retained') + code += function(source, 'apple_rtkit_management_rx_iop_pwr_ack') + code += function(source, 'apple_rtkit_wait_timeout') + MAIN + + def run(out): + out.mkdir(parents=True, exist_ok=True) + cfile, exe = out / 'retained.c', out / 'retained' + cfile.write_text(code) + subprocess.run([args.cc, '-std=gnu11', '-O1', '-g', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-parameter', '-fsanitize=address,undefined', + '-fno-sanitize-recover=all', str(cfile), '-o', str(exe)], check=True) + result = subprocess.run([str(exe)], capture_output=True, text=True) + print(result.stdout or result.stderr, end='') + return result.returncode + + if args.out: + return run(args.out) + with tempfile.TemporaryDirectory(prefix='rtkit-retained-') as out: + return run(Path(out)) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index b878404b2d9fa..2da6ddc68cd13 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -157,6 +157,7 @@ def mutations(flag, names, base, expect=KILL): 'qspi-nvram-tests.py': [(out('--output'), PASS)], 'rtkit-oslog-tests.py': [(out(), PASS)] + mutations('--mutate', ('admission', 'channel', 'counter', 'reset', 'buffer'), out()), + 'rtkit-retained-tests.py': [(out(), PASS)], 't6030-clock-tests.py': [(out(), PASS)], 't6030-display-admission-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-display-linkage-tests.py': [(out('--output') + ['--conf', '{conf}'], PASS)], diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/m4-error-paths.py b/tools/testing/selftests/drivers/gpu/apple-dcp/m4-error-paths.py index e57b10a738947..027eec2ecbc1a 100755 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/m4-error-paths.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/m4-error-paths.py @@ -59,7 +59,8 @@ def main(): votes += definition(pmp, 'enum pmp_power_kind {') + ';\n' votes += definition(pmp, 'struct pmp_power_vote {') + ';\n' for function in ('static void pmp_vote_fail(', 'static void pmp_vote_send_failed(', - 'static int pmp_vote_begin(', + 'static int pmp_vote_begin(', 'static int pmp_vote_begin_release(', + 'static int pmp_vote_check_release(', 'static int pmp_vote_finish_release(', 'static int pmp_vote_check(', 'static int pmp_vote_finish(', 'static void pmp_vote_reply('): votes += definition(pmp, function) + '\n' @@ -111,6 +112,48 @@ def main(): pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); assert(pmp_vote_check(&s, PMP_POWER_DISP) == 0); assert(pmp_vote_begin(&s, PMP_POWER_AGX) == -EPROTO); + + /* The DCP releases DISP while its domains are down and takes it again. */ + s = (struct pmp_power_vote){0}; + assert(pmp_vote_check_release(&s, PMP_POWER_DISP) == -ENODEV); /* not held */ + assert(pmp_vote_begin_release(&s, PMP_POWER_DISP) == 0); /* not held */ + assert(pmp_vote_begin(&s, PMP_POWER_DISP) == 1); + assert(pmp_vote_check_release(&s, PMP_POWER_DISP) == -ENODEV); /* still pending */ + pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); + assert(pmp_vote_check_release(&s, PMP_POWER_DISP) == 0); + assert(pmp_vote_begin_release(&s, PMP_POWER_DISP) == 1); + pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); + assert(pmp_vote_finish_release(&s, PMP_POWER_DISP) == 0); + assert(!s.held[PMP_POWER_DISP] && !s.releasing && !s.error); + assert(pmp_vote_check(&s, PMP_POWER_DISP) == -ENODEV); + assert(pmp_vote_begin(&s, PMP_POWER_DISP) == 1); + pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); + assert(pmp_vote_finish(&s, PMP_POWER_DISP) == 0); + /* One request at a time. */ + assert(pmp_vote_begin(&s, PMP_POWER_AGX) == 1); + assert(pmp_vote_check_release(&s, PMP_POWER_DISP) == -EBUSY); + assert(pmp_vote_begin_release(&s, PMP_POWER_DISP) == -EBUSY); + pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); + assert(s.held[PMP_POWER_AGX] && s.held[PMP_POWER_DISP]); + /* Not transmitted: the vote is still held. */ + assert(pmp_vote_begin_release(&s, PMP_POWER_DISP) == 1); + pmp_vote_send_failed(&s); + assert(s.held[PMP_POWER_DISP] && !s.releasing && !s.error); + /* No reply: neither held nor taken again; AGX stays held. */ + assert(pmp_vote_begin_release(&s, PMP_POWER_DISP) == 1); + assert(pmp_vote_finish_release(&s, PMP_POWER_DISP) == -ETIMEDOUT); + assert(!s.held[PMP_POWER_DISP] && !s.releasing); + assert(pmp_vote_begin(&s, PMP_POWER_DISP) == -ETIMEDOUT); + assert(pmp_vote_check(&s, PMP_POWER_AGX) == 0); + + /* Admission closed by another request: DISP stays held, but is not released. */ + s = (struct pmp_power_vote){0}; + assert(pmp_vote_begin(&s, PMP_POWER_DISP) == 1); + pmp_vote_reply(&s, PMP_ENDPOINT, PMP_POWER_REPLY); + assert(pmp_vote_begin(&s, PMP_POWER_AGX) == 1); + assert(pmp_vote_finish(&s, PMP_POWER_AGX) == -ETIMEDOUT); + assert(pmp_vote_check(&s, PMP_POWER_DISP) == 0); + assert(pmp_vote_check_release(&s, PMP_POWER_DISP) == -ETIMEDOUT); return 0; } ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py b/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py index 4d0e9c50a1082..622957c3b6837 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py @@ -493,6 +493,148 @@ def dart(root): ''' +def gate(root): + """The physical power-down and power-up of the internal DCP: their order, + the providers asked before the first step, and a late firmware message.""" + backend = (root / APPLE / 'iomfb_m4.c').read_text() + return PREFIX + r''' +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define container_of(p,t,m) ((t *)((char *)(p) - __builtin_offsetof(t,m))) +#define le32_to_cpu(x) (x) +#define dev_dbg(...) ((void)0) +#define dev_err(d,...) ((void)(d),(void)fprintf(stderr,__VA_ARGS__)) +#define spin_lock_irqsave(p,f) do { (void)(p); (f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(p); (void)(f); } while(0) +typedef u32 __le32; +typedef int64_t s64; +typedef s64 ktime_t; +static ktime_t clock_ns; +static ktime_t ktime_get(void) { return clock_ns += 1000; } +static s64 ktime_us_delta(ktime_t later,ktime_t earlier) { return (later-earlier)/1000; } +struct device { int unused; }; +struct platform_device { struct device dev; }; +struct apple_rtkit { int unused; }; +struct work_struct { int unused; }; +struct delayed_work { struct work_struct work; }; +#define to_delayed_work(w) container_of(w,struct delayed_work,work) +struct apple_dcp { bool quiescing; }; +struct dcp_m4_frame { u64 dva; }; +struct dcp_m4_transaction { unsigned calls; }; +struct iomfb_m4; +struct dcp_m4_runtime { struct iomfb_m4 *dcp; unsigned count; }; +struct iomfb_m4 { + struct apple_dcp *owner; + struct device *dev; + struct platform_device *piodma; + struct apple_rtkit *rtkit; + struct dcp_m4_runtime *runtime; + int lock,state_lock,error; + struct dcp_m4_transaction transaction; + struct dcp_m4_frame active_frame; + void *active_fb; + bool gate_capable,gated,display_off,powered_off; + struct delayed_work gate_work; + u64 gate_cycles; + s64 gate_off_us,gate_on_us; +}; +static unsigned power_gate_ms; +static struct iomfb_m4 *the_dcp; +/* Every step taken, in order. */ +static char steps[512]; +static void trace(const char *s) { strcat(steps,s); strcat(steps," "); } +static int refuse; /* the provider check that refuses (1..5), or 0 */ +static bool late_word; /* a DCP message arrives during the idle handshake */ +static void mutex_lock(int *l) { assert(!*l); *l=1; } +static void mutex_unlock(int *l) { assert(*l==1); *l=0; } +static void lockdep_assert_held(int *l) { assert(*l==1); } +static void dcp_m4_failed(struct iomfb_m4 *dcp,int error) { if(!dcp->error) dcp->error=error; } +static int dcp_m4_runtime_live(struct dcp_m4_runtime *r) { return r->dcp->error; } +static int dcp_m4_runtime_call(void *cookie,unsigned int method,bool remote, + const void *input,u32 input_size,void *output,u32 output_size) +{ + char name[16]; + (void)cookie; (void)remote; (void)input; (void)input_size; + snprintf(name,sizeof(name),"A%u",method); trace(name); + if(output) memset(output,0,output_size); + if(method==412) { assert(output_size==4); put_unaligned_le32(1,output); } + return 0; +} +static int dcp_m4_runtime_drain(struct dcp_m4_runtime *r,unsigned budget) +{ (void)r; (void)budget; trace("drain"); return 0; } +static int apple_rtkit_idle(struct apple_rtkit *rtk) { (void)rtk; trace("idle"); return 0; } +static void apple_rtkit_hold_rx(struct apple_rtkit *rtk) { (void)rtk; trace("hold"); } +static int apple_rtkit_poll(struct apple_rtkit *rtk) { (void)rtk; trace("poll"); return 0; } +static void apple_rtkit_flush_rx(struct apple_rtkit *rtk) +{ (void)rtk; trace("flush"); if(late_word) the_dcp->runtime->count++; } +static int iomfb_m4_cpu_run(struct iomfb_m4 *dcp,bool run) { (void)dcp; trace(run?"run":"stop"); return 0; } +static int iomfb_m4_cpu_start(void *data) { return iomfb_m4_cpu_run(data,true); } +static int apple_rtkit_wake_retained(struct apple_rtkit *rtk,int (*start)(void *),void *ctx) +{ (void)rtk; trace("wake"); return start(ctx); } +static bool dcp_dev(struct device *dev) { return dev==the_dcp->dev; } +static int apple_dart_dcp_power_check(struct device *dev) { return refuse==(dcp_dev(dev)?1:2)?-EAGAIN:0; } +static int apple_pmp_j713_display_check_release(struct device *dev) { (void)dev; return refuse==3?-ETIMEDOUT:0; } +static int apple_pmc_interface_consumer_check(struct device *dev) { (void)dev; return refuse==4?-ENODEV:0; } +static int apple_pmgr_ps_consumer_check(struct device *dev) { (void)dev; return refuse==5?-EBUSY:0; } +static int apple_dart_dcp_power_off(struct device *dev) { trace(dcp_dev(dev)?"dart-off":"piodma-off"); return 0; } +static int apple_dart_dcp_power_on(struct device *dev) { trace(dcp_dev(dev)?"dart-on":"piodma-on"); return 0; } +static int apple_pmp_j713_display_put(struct device *dev) { (void)dev; trace("pmp-put"); return 0; } +static int apple_pmp_j713_display_get(struct device *dev) { (void)dev; trace("pmp-get"); return 0; } +static int apple_pmc_interface_consumer_off(struct device *dev) { (void)dev; trace("pmc-off"); return 0; } +static int apple_pmc_interface_consumer_on(struct device *dev) { (void)dev; trace("pmc-on"); return 0; } +static int apple_pmgr_ps_consumer_off(struct device *dev) { (void)dev; trace("pmgr-off"); return 0; } +static int apple_pmgr_ps_consumer_on(struct device *dev) { (void)dev; trace("pmgr-on"); return 0; } +''' + '\n'.join(block(backend, name) for name in ( + 'static int iomfb_m4_gate_check(', 'static int iomfb_m4_power_gate(', + 'static int iomfb_m4_power_ungate(', 'static void iomfb_m4_gate_work(')) + r''' +int main(void) +{ + struct apple_dcp owner={0}; + struct device dev={0}; + struct platform_device piodma={{0}}; + struct apple_rtkit rtk={0}; + struct iomfb_m4 dcp={.owner=&owner,.dev=&dev,.piodma=&piodma,.rtkit=&rtk, + .gate_capable=true,.display_off=true}; + struct dcp_m4_runtime r={.dcp=&dcp}; + + dcp.runtime=&r; the_dcp=&dcp; + /* Turned off after the blank: the queued power-down does nothing. */ + iomfb_m4_gate_work(&dcp.gate_work.work); + assert(!steps[0] && !dcp.gated); + /* Wanted: the whole power-down, in its order. */ + power_gate_ms=1000; + iomfb_m4_gate_work(&dcp.gate_work.work); + assert(!strcmp(steps,"A500 drain idle hold stop poll flush dart-off piodma-off " + "pmp-put pmc-off pmgr-off ")); + assert(dcp.gated && !dcp.error && dcp.gate_cycles==1 && !dcp.lock); + + /* The power-up, in macOS's order: domains, PMP vote, PMC enables, DARTs, CPU. */ + steps[0]=0; dcp.lock=1; + assert(!iomfb_m4_power_ungate(&dcp)); + assert(!strcmp(steps,"pmgr-on pmp-get pmc-on dart-on piodma-on wake run A414 A412 ")); + assert(!dcp.gated && !dcp.error); + + /* A provider that cannot take part now: no step at all, and no failure. */ + for(refuse=1;refuse<=5;refuse++) { + steps[0]=0; + assert(iomfb_m4_power_gate(&dcp)<0); + assert(!steps[0] && !dcp.gated && !dcp.error); + } + refuse=0; + + /* A message from the idle handshake: failed with nothing powered down. */ + steps[0]=0; late_word=true; + assert(iomfb_m4_power_gate(&dcp)==-EBUSY); + assert(!strcmp(steps,"A500 drain idle hold stop poll flush ")); + assert(!dcp.gated && dcp.error==-EBUSY); + dcp.lock=0; + puts("PASS the power-down and power-up keep their order, a refusing provider changes nothing, " + "and a late message fails before anything is powered down"); + return 0; +} +''' + + def blank(root): backend = (root / APPLE / 'iomfb_m4.c').read_text() backlight = (root / APPLE / 'dcp_backlight.c').read_text() @@ -551,7 +693,16 @@ def blank(root): bool display_off; u64 display_off_cycles; s64 display_off_us,display_off_us_max,display_on_us,display_on_us_max; + bool gate_capable,gated; + int gate_work; }; +/* The power-down after a blank: its scheduling and the power-up before ON. */ +static unsigned power_gate_ms,scheduled,cancelled,ungates,ungate_error,ons_at_ungate; +#define system_dfl_wq NULL +static unsigned long msecs_to_jiffies(unsigned ms) { return ms; } +static bool mod_delayed_work(void *wq,int *work,unsigned long delay) +{ (void)wq; (void)work; assert(delay==power_gate_ms); scheduled++; return true; } +static bool cancel_delayed_work(int *work) { (void)work; cancelled++; return false; } static void mutex_lock(int *lock) { assert(!*lock); *lock=1; } static void mutex_unlock(int *lock) { assert(*lock==1); *lock=0; } static void lockdep_assert_held(int *lock) { assert(*lock==1); } @@ -568,6 +719,13 @@ def blank(root): static int dcp_m4_set_power_state(struct iomfb_m4 *dcp,bool on) { assert(dcp->lock==1); if(on) power_ons++; else power_offs++; return 0; } static int dcp_m4_runtime_live(struct dcp_m4_runtime *r) { return r->dcp->error; } +static int iomfb_m4_power_ungate(struct iomfb_m4 *dcp) +{ + assert(dcp->lock==1 && dcp->gated && dcp->display_off); + ons_at_ungate=power_ons; ungates++; + if(ungate_error) { dcp_m4_failed(dcp,ungate_error); return ungate_error; } + dcp->gated=false; return 0; +} static int iomfb_m4_check_frame(const struct dcp_m4_frame *f) { return dcp_m4_frame_check(f); } static unsigned order,failure,publications; @@ -640,6 +798,42 @@ def blank(root): assert(fb.refs==2 && next.refs==2 && dcp.active_fb==&fb && dcp.uncertain_fb==&next); failure=0; } + /* Not opted in: a blank schedules no power-down; a frame still cancels one. */ + { + struct drm_format_info fmt={.format=DRM_FORMAT_XRGB8888,.num_planes=1}; + struct drm_framebuffer fb={.format=&fmt,.width=2560,.height=1664,.pitches={10240},.refs=1}; + struct apple_dcp owner={0}; + struct iomfb_m4 dcp={.owner=&owner}; + struct dcp_m4_runtime runtime={.dcp=&dcp}; + unsigned c; + owner.v26_6=&dcp; dcp.runtime=&runtime; + assert(!dcp_bl_init(&owner.backlight.state,500,true,120,false,0)); + c=cancelled; + assert(!iomfb_m4_present(&owner,&fb,0x10000100000ULL,0x2000000)); + assert(cancelled==c+1); + assert(!iomfb_m4_blank(&owner) && scheduled==0); + dcp.gate_capable=true; + assert(!iomfb_m4_present(&owner,&fb,0x10000100000ULL,0x2000000)); + assert(!iomfb_m4_blank(&owner) && scheduled==0); /* power_gate_ms is 0 */ + + /* Opted in: one power-down after the blank, after power_gate_ms. */ + power_gate_ms=1000; + assert(!iomfb_m4_present(&owner,&fb,0x10000100000ULL,0x2000000)); + assert(!iomfb_m4_blank(&owner) && scheduled==1); + /* Powered down: the next frame powers it up first, then setPowerState(1). */ + dcp.gated=true; + unsigned ons=power_ons; + assert(!iomfb_m4_present(&owner,&fb,0x10000100000ULL,0x2000000)); + assert(ungates==1 && ons_at_ungate==ons && power_ons==ons+1); + assert(!dcp.gated && !dcp.display_off && dcp.active_fb==&fb); + /* A failed power-up refuses the frame; the session is failed by it. */ + assert(!iomfb_m4_blank(&owner) && scheduled==2); + dcp.gated=true; ungate_error=-EIO; ons=power_ons; + assert(iomfb_m4_present(&owner,&fb,0x10000100000ULL,0x2000000)==-EIO); + assert(ungates==2 && power_ons==ons && dcp.error==-EIO && dcp.display_off); + assert(fb.refs==1 && !dcp.active_fb); + ungate_error=0; power_gate_ms=0; + } puts("PASS DPMS-off blank carries zero before delayed work and completes only after both exact receipts"); puts("PASS actual present/blank retain active+uncertain DMA framebuffer references across a missing receipt"); puts("PASS a present while the display is powered off for sleep is refused without a reference"); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index ac69a16514c3e..18374e2337a58 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -83,6 +83,7 @@ def main(): ('m4', 'm4-common-plane', m4.plane, ((),), ('-Wno-unused-function', '-Wno-unused-const-variable')), ('m4', 'm4-producer-retirement', m4.prepare, ((),), ('-Wno-unused-function',)), ('m4', 'm4-dpms-retirement', m4.blank, ((),), ('-Wno-unused-function', '-Wno-unused-const-variable')), + ('m4', 'm4-power-gate', m4.gate, ((),), ('-Wno-unused-function',)), ('m4', 'm4-dart-consumer', m4.dart, ((),), ('-Wno-unused-function',)), ('m4', 'm4-panel-timing', m4.timing, ((),), ('-Wno-unused-function', '-Wno-unused-const-variable')), ('boot', 'oslog-events', boot.oslog, ((),), ()),