diff --git a/Documentation/ABI/testing/sysfs-driver-sn201202x b/Documentation/ABI/testing/sysfs-driver-sn201202x new file mode 100644 index 0000000000000..39d8040b6d32d --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-sn201202x @@ -0,0 +1,25 @@ +What: /sys/bus/spmi/devices//hib_resume +Date: November 2026 +KernelVersion: 7.4 +Contact: František Balázsy +Description: + (RW) How the last resume from hibernation left an Apple + SN201202x USB Type-C port controller. Reads one of: + + none + No resume from hibernation since the last system + sleep, or "roles" was written since. + roles-pending + The controller was restored in place after the cold + boot behind the resume. The kernel revalidates the + port, without taking the port or a display on it + down, once the USB controller that provides its role + switch is bound. + failed + The in-place restore failed; the controller has to be + probed again (unbind and bind the device). + + Writing "roles" while the state is "roles-pending" takes note + of it: the state then reads "none". The write is not needed + for the port to be revalidated. Other values fail with + -EINVAL, a write in another state with -EALREADY. diff --git a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml index 91ae5a3ce1bca..4c736d06122dd 100644 --- a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml +++ b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml @@ -53,6 +53,8 @@ properties: reg: maxItems: 1 + nonposted-mmio: true + "#address-cells": const: 1 diff --git a/Documentation/devicetree/bindings/chosen/aura,hibernate.yaml b/Documentation/devicetree/bindings/chosen/aura,hibernate.yaml new file mode 100644 index 0000000000000..8c8e9390b1e60 --- /dev/null +++ b/Documentation/devicetree/bindings/chosen/aura,hibernate.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/chosen/aura,hibernate.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Memory layout a boot object keeps the same on every boot + +maintainers: + - František Balázsy + +description: | + A hibernation image puts every saved page back into the frame it was taken + from, so the boot that resumes an image has to give the kernel the same RAM, + and keep the same ranges from it, as the boot that wrote the image. On Apple + silicon the boot firmware loads the boot object at an address that changes + from boot to boot, and what the boot object keeps alive after the hand-off + moves with it. A boot object that instead keeps everything from the start of + RAM up to a fixed end from the kernel, at the same addresses on every boot, + says so with aura,hibernate-layout. Without it the kernel neither hibernates + nor resumes. + + The boot firmware still places some firmware segments anew on every boot, + inside that range, and a kernel restored from an image holds the device tree + of the boot that wrote the image. The boot object therefore puts the device + tree of every boot at one address inside the range, given in + aura,hibernate-dt. The image does not contain that memory, so the restored + kernel reads there what the resuming boot placed. + + The properties are named after aura, the boot loader that defined them and + whose released versions publish them. + +select: + properties: + $nodename: + const: chosen + required: + - $nodename + +properties: + $nodename: + const: chosen + + aura,hibernate-layout: + $ref: /schemas/types.yaml#/definitions/uint64-array + description: + The range of RAM that is kept from the kernel and laid out the same on + every boot. + items: + - description: Physical address of the start of RAM. + - description: Physical address of the end of the range, exclusive. + + aura,hibernate-dt: + $ref: /schemas/types.yaml#/definitions/uint64 + description: + Physical address, inside the range of aura,hibernate-layout, of the + device tree the kernel is started with. The kernel reads at most 2 MiB + there. + +dependencies: + aura,hibernate-dt: [ 'aura,hibernate-layout' ] + +additionalProperties: true +... diff --git a/Documentation/devicetree/bindings/chosen/aurora,s2r-mailbox.yaml b/Documentation/devicetree/bindings/chosen/aurora,s2r-mailbox.yaml new file mode 100644 index 0000000000000..a48f9bbc5ae3e --- /dev/null +++ b/Documentation/devicetree/bindings/chosen/aurora,s2r-mailbox.yaml @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/chosen/aurora,s2r-mailbox.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Return path of a boot object that owns the CPU reset vectors + +maintainers: + - František Balázsy + +description: | + On Apple silicon the boot firmware locks every CPU's reset vector to the + boot object it started. A CPU that comes back from a power-off, and every + CPU after a suspend to RAM, therefore restarts in the boot object, not in the + kernel. A boot object that sends such a CPU back to the kernel says so with + this property: a secondary CPU returns through its spin-table release + address, and the boot CPU of a resume jumps, at EL2 with the MMU off, to the + address the kernel left in the mailbox. Its presence also tells the kernel + that its spin-table CPUs may leave the kernel and come back. + + The mailbox is four 64-bit words in the CPU's byte order: + 0: the magic value 0x3152325341525541 ("AURAS2R1"), written by the boot + object before the kernel starts + 1: the MPIDR of the only CPU that may take the resume address + 2: the resume address, set by the kernel before it suspends and cleared + by the boot object when a CPU takes it + 3: how many times the resume address was taken + +select: + properties: + $nodename: + const: chosen + required: + - $nodename + +properties: + $nodename: + const: chosen + + aurora,s2r-mailbox: + $ref: /schemas/types.yaml#/definitions/uint64 + description: + Physical address of the mailbox, inside the boot object's own memory, + which the kernel does not use. + +additionalProperties: true +... diff --git a/Documentation/devicetree/bindings/display/apple,t6041-display-power.yaml b/Documentation/devicetree/bindings/display/apple,t6041-display-power.yaml new file mode 100644 index 0000000000000..ab19f769ea9f4 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,t6041-display-power.yaml @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,t6041-display-power.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T6040 display power words + +maintainers: + - František Balázsy + +description: | + The words a T6040 (M4 Pro) display coprocessor's driver writes to lower + its power domains after it stops the firmware for a suspend-to-RAM and to + raise them again on the wake. A display coprocessor names this node in its + apple,display-power property. The words belong to the power manager and + the display block, so no driver binds it. + + The internal display coprocessor has its power-state words in the SoC's + east power manager block, the external ones in the west block, each at its + own offsets; the display PMC word that counts the powered coprocessors is + shared by all three. The boot loader publishes one node for the internal + coprocessor and one that both external coprocessors name. + +properties: + compatible: + const: apple,t6041-display-power + + reg: + items: + - description: a register window, named by reg-names + - description: a register window, named by reg-names + + reg-names: + description: + pmc-dcp is the display PMC word that counts the powered coprocessors, + ps the power manager block that holds the coprocessors' power-state + words. The two may come in either order. + oneOf: + - items: + - const: pmc-dcp + - const: ps + - items: + - const: ps + - const: pmc-dcp + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + display-power@5048a9c14 { + compatible = "apple,t6041-display-power"; + reg = <0x5 0x048a9c14 0x0 0x8>, + <0x5 0x02280000 0x0 0x14000>; + reg-names = "pmc-dcp", "ps"; + }; + + display-power@502800000 { + compatible = "apple,t6041-display-power"; + reg = <0x5 0x02800000 0x0 0x14000>, + <0x5 0x048a9c14 0x0 0x8>; + reg-names = "ps", "pmc-dcp"; + }; + }; diff --git a/Documentation/devicetree/bindings/media/apple,prores.yaml b/Documentation/devicetree/bindings/media/apple,prores.yaml index 69be59afd4e70..827863013b202 100644 --- a/Documentation/devicetree/bindings/media/apple,prores.yaml +++ b/Documentation/devicetree/bindings/media/apple,prores.yaml @@ -248,8 +248,12 @@ allOf: maxItems: 1 iommus: maxItems: 1 + resets: + description: + The shared codec reset. It also clears the + translations of the engine's DART, so a board may + leave it out and have a hung engine refused instead. required: - - resets - reg-names - if: properties: diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 55d40f9a40cc7..dea7da061a73f 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -504,7 +504,7 @@ allOf: properties: reg: minItems: 8 - maxItems: 8 + maxItems: 9 interrupts: minItems: 6 maxItems: 6 diff --git a/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml index 3178de1a8b7e4..68ee0945b02d0 100644 --- a/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml @@ -172,6 +172,29 @@ properties: $ref: "#/$defs/apple,tunable" description: USB2 host tunables, applied to the bank0 region. + apple,tunable-cio-lane0: + $ref: "#/$defs/apple,tunable" + description: > + Lane 0 receiver defaults for CIO (USB4/Thunderbolt) mode, applied to the + core region when the port enters USB4 or Thunderbolt. CIO mode is offered + only when both lane tables are present. + + apple,tunable-cio-lane1: + $ref: "#/$defs/apple,tunable" + description: Lane 1 receiver defaults for CIO mode, applied to the core region. + + apple,tunable-cio-uc-regs: + $ref: "#/$defs/apple,tunable" + description: Optional PHY microcontroller defaults for CIO mode, core region. + + apple,tunable-cio-lane0-eq: + $ref: "#/$defs/apple,tunable" + description: Optional lane 0 receiver equalization for CIO mode, core region. + + apple,tunable-cio-lane1-eq: + $ref: "#/$defs/apple,tunable" + description: Optional lane 1 receiver equalization for CIO mode, core region. + required: - compatible - reg @@ -199,6 +222,11 @@ allOf: mode-switch: false port: false ports: false + apple,tunable-cio-lane0: false + apple,tunable-cio-lane1: false + apple,tunable-cio-uc-regs: false + apple,tunable-cio-lane0-eq: false + apple,tunable-cio-lane1-eq: false additionalProperties: false diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml index e2bec4d954de9..228151a45575d 100644 --- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml @@ -69,6 +69,14 @@ properties: $ref: /schemas/types.yaml#/definitions/uint32 description: The number of pins in this GPIO controller. + apple,resume-restore-all: + type: boolean + description: + After a sleep that powers the block down, every pin's configuration is + written back, not only the pins a driver has claimed, as the vendor + firmware does for this block. The pins in gpio-reserved-ranges are left + as the wake found them. + gpio-reserved-ranges: true interrupts: diff --git a/Documentation/devicetree/bindings/soc/apple/apple,sleep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,sleep.yaml new file mode 100644 index 0000000000000..d084121d3bfb1 --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,sleep.yaml @@ -0,0 +1,262 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,sleep.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC system sleep controls + +maintainers: + - Diljit Singh + - František Balázsy + +description: | + The registers a kernel running directly at EL2 on an Apple SoC writes to enter + and return from suspend-to-RAM and to power its secondary CPUs off. They belong + to several hardware blocks, some of which other drivers also use, so this node + overlaps those drivers' register ranges and the driver does not claim them. + Each SoC has its own compatible, as its power domains differ. Blocks that + several SoCs have are named alike. + + On the T6040 the sleep engine waits for the display pipes, so the node lists + the power states that have to be off before a sleep, as the ADT's + device-check-ps-b4-sleep table does. Each entry names a PMGR register block, + the offset of a display pipe's CPU power state and the offset of its last + sub-block; the states in between are 8 bytes apart. All of the T6040's + register blocks are needed to enter a suspend and to undo it. + +properties: + compatible: + enum: + - apple,t6040-sleep + - apple,t8132-sleep + - apple,t8140-sleep + + reg: + minItems: 11 + maxItems: 28 + + reg-names: + minItems: 11 + maxItems: 28 + + apple,wake-enable: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The wake source IDs left enabled across suspend-to-RAM. Every other + source is masked. + + apple,apcie: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The PCIe host whose shared registers and tunables the storage controller's + link depends on. They are needed even with that host disabled. On the + T6040 the boot loader adds it, with those registers and tunables on the + host's node; without it the storage controller can't be stopped for a + sleep. + + apple,pinctrl: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The pin controller whose pins a deep sleep resets. Its pins are saved + before a sleep and put back after it, before the storage controller + starts. + + apple,pinctrl-skip-pins: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + Pins of that controller to leave alone across a sleep, neither saved nor + put back. + + apple,debug-port: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The USB-C port controller that carries the debug link, stopped for a sleep. + + apple,display-coprocessors: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + The DCP and the external DCPs, which must be off before a commit, in that + order. + + apple,check-ps-b4-sleep: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + The display pipes whose power states have to be off before a sleep, one + entry per pipe. + items: + items: + - description: PMGR syscon + - description: offset of the pipe's CPU power state + - description: offset of its last sub-block's power state + + apple,check-ps-b4-sleep-names: + $ref: /schemas/types.yaml#/definitions/string-array + description: One name per apple,check-ps-b4-sleep entry. + +required: + - compatible + +allOf: + - if: + properties: + compatible: + const: apple,t6040-sleep + then: + properties: + reg: + items: + - description: CPU start/stop block (pmgr + 0x88000) + - description: AON watchdog + - description: control words of the three CPU complexes + - description: SoC power gate, wake and retention voters + - description: clock generator, the clock mux park words + - description: SPMI controller of the PMU (boot stage, notify, hibernate) + - description: second SPMI controller's notify word + - description: MasterMini block with the S2R timer + - description: SOC-East page with retention, fabric, DCS, SIO and other power words + - description: SOC page with the west axi2af, ATC, fabric and DCS power words + - description: AOP global block, S2R control and wake masks + reg-names: + items: + - const: cpu-start + - const: wdt + - const: cluster + - const: pwrgate + - const: clkgen + - const: spmi + - const: spmi-a0 + - const: mastermini + - const: soc-east + - const: soc-ps + - const: wake + apple,wake-enable: false + apple,pinctrl: false + apple,pinctrl-skip-pins: false + apple,debug-port: false + apple,display-coprocessors: false + required: + - reg + - reg-names + - apple,check-ps-b4-sleep + - apple,check-ps-b4-sleep-names + + - if: + properties: + compatible: + const: apple,t8132-sleep + then: + properties: + reg: + minItems: 25 + maxItems: 26 + reg-names: + minItems: 25 + items: + - const: ps + - const: pwrgate + - const: cpu-start + - const: cllt + - const: cluster + - const: commit + - const: wdt + - const: aon-misc + - const: wake + - const: amcc0 + - const: amcc1 + - const: pmc-dcp + - const: dvfm-e + - const: dvfm-p + - const: spmi + - const: pmc-voters + - const: dcs + - const: ps-aon + - const: spmi-a0 + - const: clk-mux + - const: mcc0 + - const: mcc1 + - const: ptd + - const: clk-perf + - const: cluster-dcs + - const: ps-prot + apple,check-ps-b4-sleep: false + apple,check-ps-b4-sleep-names: false + required: + - reg + - reg-names + - apple,display-coprocessors + + - if: + properties: + compatible: + const: apple,t8140-sleep + then: + properties: + reg: + minItems: 28 + reg-names: + items: + - const: ps + - const: pwrgate + - const: cpu-start + - const: cllt + - const: cluster + - const: commit + - const: wdt + - const: aon-misc + - const: wake + - const: amcc0 + - const: amcc1 + - const: amcc2 + - const: amcc3 + - const: pmc-dcp + - const: dvfm-e + - const: dvfm-p + - const: spmi + - const: pmc-voters + - const: dcs + - const: ps-aon + - const: spmi-a0 + - const: clk-mux + - const: mcc0 + - const: mcc1 + - const: mcc2 + - const: ptd + - const: clk-perf + - const: cluster-dcs + apple,check-ps-b4-sleep: false + apple,check-ps-b4-sleep-names: false + required: + - reg + - reg-names + - apple,wake-enable + - apple,display-coprocessors + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + sleep-controller@502888000 { + compatible = "apple,t6040-sleep"; + reg = <0x5 0x02888000 0x0 0x20>, + <0x5 0x0836c000 0x0 0x4000>, + <0x5 0x02820000 0x0 0x20>, + <0x5 0x0282c000 0x0 0x1000>, + <0x5 0x00000000 0x0 0x80000>, + <0x5 0x09014000 0x0 0x1000>, + <0x5 0x09188000 0x0 0x1000>, + <0x5 0x08900000 0x0 0x8000>, + <0x5 0x02280000 0x0 0xd000>, + <0x5 0x02800000 0x0 0x9000>, + <0x5 0x08004000 0x0 0x1000>; + reg-names = "cpu-start", "wdt", "cluster", "pwrgate", + "clkgen", "spmi", "spmi-a0", "mastermini", + "soc-east", "soc-ps", "wake"; + apple,check-ps-b4-sleep = <&pmgr_disp 0x0 0x58>; + apple,check-ps-b4-sleep-names = "disp"; + }; + }; diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml deleted file mode 100644 index bdbe2a5ccadaf..0000000000000 --- a/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml +++ /dev/null @@ -1,158 +0,0 @@ -# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/soc/apple/apple,t8140-sleep.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Apple SoC system sleep controls - -maintainers: - - Diljit Singh - -description: | - The registers a kernel running directly at EL2 on an Apple SoC writes to enter - and return from suspend-to-RAM and to power its secondary CPUs off. They belong - to several hardware blocks, some of which other drivers also use, so this node - overlaps those drivers' register ranges and the driver does not claim them. - Each SoC has its own compatible, as its power domains differ. - -properties: - compatible: - enum: - - apple,t8132-sleep - - apple,t8140-sleep - - reg: - minItems: 25 - maxItems: 28 - - reg-names: - minItems: 25 - maxItems: 28 - - apple,wake-enable: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: - The wake source IDs left enabled across suspend-to-RAM. Every other - source is masked. - - apple,apcie: - $ref: /schemas/types.yaml#/definitions/phandle - description: - The PCIe host whose shared registers and tunables the storage controller's - link depends on. They are needed even with that host disabled. - - apple,pinctrl: - $ref: /schemas/types.yaml#/definitions/phandle - description: - The pin controller whose pins a deep sleep resets. Its pins are saved - before a sleep and put back after it, before the storage controller - starts. - - apple,pinctrl-skip-pins: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: - Pins of that controller to leave alone across a sleep, neither saved nor - put back. - - apple,debug-port: - $ref: /schemas/types.yaml#/definitions/phandle - description: - The USB-C port controller that carries the debug link, stopped for a sleep. - - apple,display-coprocessors: - $ref: /schemas/types.yaml#/definitions/phandle-array - description: - The DCP and the external DCPs, which must be off before a commit, in that - order. - -required: - - compatible - - reg - - reg-names - - apple,display-coprocessors - -allOf: - - if: - properties: - compatible: - contains: - const: apple,t8140-sleep - then: - required: - - apple,wake-enable - properties: - reg: - minItems: 28 - maxItems: 28 - reg-names: - items: - - const: ps - - const: pwrgate - - const: cpu-start - - const: cllt - - const: cluster - - const: commit - - const: wdt - - const: aon-misc - - const: wake - - const: amcc0 - - const: amcc1 - - const: amcc2 - - const: amcc3 - - const: pmc-dcp - - const: dvfm-e - - const: dvfm-p - - const: spmi - - const: pmc-voters - - const: dcs - - const: ps-aon - - const: spmi-a0 - - const: clk-mux - - const: mcc0 - - const: mcc1 - - const: mcc2 - - const: ptd - - const: clk-perf - - const: cluster-dcs - - if: - properties: - compatible: - contains: - const: apple,t8132-sleep - then: - properties: - reg: - minItems: 25 - maxItems: 26 - reg-names: - minItems: 25 - items: - - const: ps - - const: pwrgate - - const: cpu-start - - const: cllt - - const: cluster - - const: commit - - const: wdt - - const: aon-misc - - const: wake - - const: amcc0 - - const: amcc1 - - const: pmc-dcp - - const: dvfm-e - - const: dvfm-p - - const: spmi - - const: pmc-voters - - const: dcs - - const: ps-aon - - const: spmi-a0 - - const: clk-mux - - const: mcc0 - - const: mcc1 - - const: ptd - - const: clk-perf - - const: cluster-dcs - - const: ps-prot - -additionalProperties: false diff --git a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml index 78dcee199f937..c730e4f46a020 100644 --- a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml @@ -50,6 +50,8 @@ properties: power-domains: maxItems: 1 + post-init-providers: true + ports: $ref: /schemas/graph.yaml#/properties/ports properties: diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index 562fb2f315e5c..b51948af8b09f 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -124,7 +124,8 @@ properties: description: The Type-C PHY whose lanes carry the router's CIO link. A host that power-cycles the router for a deep sleep takes the PHY through the - sleep with it. + sleep with it. On T6040 the router starts only while that PHY is in + CIO mode and tells it when it stops. phy-names: const: cio @@ -285,6 +286,8 @@ allOf: - nhi@1f00000 - interrupts - interrupt-names + - phys + - phy-names else: required: - iommu@a80000 diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 4775b267e1f77..ed0d32cde3614 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -207,6 +207,8 @@ patternProperties: description: Atmel Corporation "^auo,.*": description: AU Optronics Corporation + "^aura,.*": + description: aura boot loader for Apple silicon "^aurora,.*": description: Aurora Silicon "^auvidea,.*": diff --git a/MAINTAINERS b/MAINTAINERS index 3f919f3e327f3..70fe19771d80e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2093,7 +2093,9 @@ F: drivers/soc/apple/pmp.rs F: include/linux/mt7932.h F: include/linux/soc/apple/pmp-report.h F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml +F: Documentation/devicetree/bindings/chosen/aura,hibernate.yaml F: Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml +F: Documentation/devicetree/bindings/chosen/aurora,s2r-mailbox.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml @@ -2110,10 +2112,10 @@ F: Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,sleep.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml -F: Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml F: Documentation/devicetree/bindings/sound/apple,t6041-aop-audio.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml @@ -2136,6 +2138,7 @@ F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c F: drivers/soc/apple/sleep*.c +F: drivers/soc/apple/sleep-internal.h F: drivers/tty/serial/apple_dockchannel* F: include/linux/soc/apple/pci-apple-piodma.h F: include/linux/soc/apple/sleep.h diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 81f92a6fb50f7..224bba9710481 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2689,6 +2689,10 @@ config ARCH_HIBERNATION_HEADER def_bool y depends on HIBERNATION +config ARCH_HIBERNATION_CLEAN_IMAGE + def_bool y + depends on HIBERNATION + config ARCH_SUSPEND_POSSIBLE def_bool y diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index e81fb8b6b5d0d..4cf87185fe811 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -71,10 +71,24 @@ adj-height-mm = <216>; }; +/* + * The AOP firmware reports the lid angle to the SMC once a second on its own, + * and every report on this mailbox ends a suspend-to-idle period: keep its + * messages in the FIFO across system sleep. + */ +&aop_mbox { + apple,hold-rx-in-sleep; +}; + &aop_audio { apple,chassis-name = "J616"; }; +/* Suspend-to-RAM through the boot object has been tested on this board. */ +&t6040_sleep { + status = "okay"; +}; + &sound { compatible = "apple,j616-macaudio", "apple,macaudio"; model = "MacBook Pro J616"; @@ -388,3 +402,102 @@ &aes { status = "okay"; }; + +/* + * The ProRes encoder: the shared codec power state (below the always-on + * fabric domain), its DART and the PMP soc-device record SCODEC0 (27). + * No reset: the shared codec's reset also clears the DART's translations, + * so the driver refuses a hung engine instead. + */ +&ps_fab1_soc { + status = "okay"; +}; + +&ps_scodec { + status = "okay"; +}; + +&pmp_report { + pmp_report_scodec: report@1b { + compatible = "apple,t6041-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1b>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + }; +}; + +&apr0_dart { + status = "okay"; +}; + +&apr0 { + power-domains = <&ps_scodec>, <&pmp_report_scodec>; + power-domain-names = "core", "perf"; + /delete-property/ resets; + status = "okay"; +}; + +/* + * The JPEG engines share the PMP soc-device record JPEG0 (26): hold it as + * their performance request while an engine runs. + */ +&pmp_report { + pmp_report_jpg: report@1a { + compatible = "apple,t6041-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1a>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + }; +}; + +&jpeg0 { + power-domains = <&ps_jpg>, <&pmp_report_jpg>; + power-domain-names = "core", "perf"; +}; + +&jpeg1 { + power-domains = <&ps_jpg>, <&pmp_report_jpg>; + power-domain-names = "core", "perf"; +}; + +/* + * DP/HDMI audio: the clients behind dcpext0/dcpext1, their power states and + * the SIO the boot loader publishes. Neither side may hold the other at + * probe: audio needs the SIO only when a stream opens, and the external + * displays must come up whether the audio clients bind or not. Only + * properties are added to the display processors: their status belongs to + * the display gate. + */ +&ps_dpa_p { + status = "okay"; +}; + +&ps_dpa1 { + status = "okay"; +}; + +&ps_dpa2 { + status = "okay"; +}; + +&dpaudio1 { + post-init-providers = <&sio>; + status = "okay"; +}; + +&dpaudio2 { + post-init-providers = <&sio>; + status = "okay"; +}; + +&dcpext0 { + post-init-providers = <&dpaudio1>; +}; + +&dcpext1 { + post-init-providers = <&dpaudio2>; +}; diff --git a/arch/arm64/boot/dts/apple/t6040-usb4.dtsi b/arch/arm64/boot/dts/apple/t6040-usb4.dtsi index cd3b2c21aca7a..cf75493f031b3 100644 --- a/arch/arm64/boot/dts/apple/t6040-usb4.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-usb4.dtsi @@ -36,6 +36,9 @@ pinctrl-names = "lstx"; pinctrl-0 = <&usb4_0_lstx_pins>; lstx-gpios = <&pinctrl_aop 49 GPIO_ACTIVE_HIGH>; + /* The router starts only over its port's Type-C PHY in CIO mode. */ + phys = <&atcphy0 PHY_TYPE_USB2>; + phy-names = "cio"; power-domains = <&ps_atc0_cio>, <&ps_atc0_cio_pcie>, <&ps_atc0_cio_usb>, <&ps_atc0_pcie>; thunderbolt-switch; @@ -133,6 +136,9 @@ pinctrl-names = "lstx"; pinctrl-0 = <&usb4_1_lstx_pins>; lstx-gpios = <&pinctrl_aop 51 GPIO_ACTIVE_HIGH>; + /* The router starts only over its port's Type-C PHY in CIO mode. */ + phys = <&atcphy1 PHY_TYPE_USB2>; + phy-names = "cio"; power-domains = <&ps_atc1_cio>, <&ps_atc1_cio_pcie>, <&ps_atc1_cio_usb>, <&ps_atc1_pcie>; thunderbolt-switch; @@ -230,6 +236,9 @@ pinctrl-names = "lstx"; pinctrl-0 = <&usb4_2_lstx_pins>; lstx-gpios = <&pinctrl_aop 53 GPIO_ACTIVE_HIGH>; + /* The router starts only over its port's Type-C PHY in CIO mode. */ + phys = <&atcphy2 PHY_TYPE_USB2>; + phy-names = "cio"; power-domains = <&ps_atc2_cio>, <&ps_atc2_cio_pcie>, <&ps_atc2_cio_usb>, <&ps_atc2_pcie>; thunderbolt-switch; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 2cc327c7a2f8f..37ce82fc38171 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -9,6 +9,10 @@ /* Blocks and properties the T6041 tree does not describe yet. */ / { + aliases { + sio = &sio; + }; + opp_p0: opp-table-p0 { compatible = "operating-points-v2"; opp-shared; @@ -241,9 +245,70 @@ pmgr_disp: power-management@502290000 { compatible = "apple,t6041-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; reg = <0x5 0x02290000 0x0 0x4000>; + /* DISP_CPU's power word takes non-posted accesses (nGnRnE). */ + nonposted-mmio; #address-cells = <1>; #size-cells = <1>; }; + + t6040_sleep: sleep-controller@502888000 { + compatible = "apple,t6040-sleep"; + /* + * The SoC blocks the commit and wake + * write, all shared read/write with their owning drivers and + * touched only once no blocker remains: + * cpu-start pmgr reg0 + 0x88000 (misc-cores): CPU start/stop + * wdt the AON watchdog, the sleep's WD2 safety net + * cluster 0x502820000: the three CPU complexes E/P0/P1 + * pwrgate 0x50282c000: wake + CPM retention voters + * clkgen 0x500000000: the clk-mux park words + * spmi 0x509014000: the PMU boot-stage + NOTIFY + go-hib + * spmi-a0 0x509188000: the second SPMI NOTIFY + * mastermini 0x508900000: the S2R timer at +0x4000 (the commit) + * soc-east 0x502280000: SOC-East page, retention + east power words + * soc-ps 0x502800000: SOC page, the west axi2af power words + * wake 0x508004000: the S2R control + wake masks/status + * The handler refuses before touching any of them. + */ + reg = <0x5 0x02888000 0x0 0x20>, + <0x5 0x0836c000 0x0 0x4000>, + <0x5 0x02820000 0x0 0x20>, + <0x5 0x0282c000 0x0 0x1000>, + <0x5 0x00000000 0x0 0x80000>, + <0x5 0x09014000 0x0 0x1000>, + <0x5 0x09188000 0x0 0x1000>, + <0x5 0x08900000 0x0 0x8000>, + <0x5 0x02280000 0x0 0xd000>, + <0x5 0x02800000 0x0 0x9000>, + <0x5 0x08004000 0x0 0x1000>; + reg-names = "cpu-start", "wdt", "cluster", "pwrgate", + "clkgen", "spmi", "spmi-a0", "mastermini", + "soc-east", "soc-ps", "wake"; + /* + * ADT pmgr device-check-ps-b4-sleep (check-ps-b4-sleep = 0x1f): per + * display pipe its CPU and its last sub-block, 8 bytes apart. + */ + apple,check-ps-b4-sleep = <&pmgr_disp 0x0 0x58>, <&pmgr_aux 0x350 0x388>, + <&pmgr_aux 0x4a0 0x4d8>, <&pmgr_aux 0x4f0 0x528>, + <&pmgr_aux 0x2e8 0x320>; + apple,check-ps-b4-sleep-names = "disp", "dispext0", "dispext1", "dispext2", + "dispext3"; + /* + * apple,apcie, for the sleep of ANS's storage PCIe link (two ST lanes + * and a shared PHY), comes from the boot loader. From the ADT's + * /arm-io/apcie0 it adds it here and puts the axi2af, cio3pll, clkgen, + * phy-block and phy-ip windows and the apple,tunable-* arrays on the + * internal PCIe host (apple,t6020-pcie, the one with an "rc" + * reg-name), in the single-lane T8140 shape. It leaves a node that + * already has apple,apcie alone, so this tree must give neither + * apple,apcie nor an apcie-storage node. Without it a sleep with ANS + * stopped is refused, and suspend-to-idle stays the default. The ST + * power words (ps_ans, ps_apcie_st0/st1, ps_apcie_sys_st0/st1) are in + * SOC-East and are driven through the soc-east window. + */ + /* Only on a board whose sleep has been tested. */ + status = "disabled"; + }; }; &pmc { diff --git a/arch/arm64/boot/dts/apple/t604x.dtsi b/arch/arm64/boot/dts/apple/t604x.dtsi index 3484262fa7a43..a032ea8b3b07f 100644 --- a/arch/arm64/boot/dts/apple/t604x.dtsi +++ b/arch/arm64/boot/dts/apple/t604x.dtsi @@ -1370,6 +1370,10 @@ #gpio-cells = <2>; gpio-ranges = <&pinctrl 0 0 200>; apple,npins = <200>; + /* ADT gpio0: every pin is put back after a sleep, except no-restore-gpio-pins. */ + apple,resume-restore-all; + /* Pins the firmware lists as no-restore */ + gpio-reserved-ranges = <190 3>; interrupt-controller; #interrupt-cells = <2>; diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index 4ed50e0dc0b5c..6626229cbec35 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -66,6 +66,12 @@ struct spin_table_power_ops { #ifdef CONFIG_HOTPLUG_CPU int spin_table_set_power_ops(const struct spin_table_power_ops *ops); +#ifdef CONFIG_HIBERNATION +int spin_table_hibernate_pen_begin(phys_addr_t image_ttbr1, int primary); +void spin_table_hibernate_pen_end(bool frozen); +void spin_table_hibernate_restored(void); +bool spin_table_hibernate_cpus_stuck(void); +#endif int spin_table_hold_parked(void); void spin_table_unhold_parked(void); int spin_table_stop_parked(void); @@ -99,6 +105,19 @@ static inline bool spin_table_cpus_stuck(void) } #endif +#if !defined(CONFIG_HOTPLUG_CPU) || !defined(CONFIG_HIBERNATION) +static inline int spin_table_hibernate_pen_begin(phys_addr_t image_ttbr1, int primary) +{ + return 0; +} +static inline void spin_table_hibernate_pen_end(bool frozen) {} +static inline void spin_table_hibernate_restored(void) {} +static inline bool spin_table_hibernate_cpus_stuck(void) +{ + return false; +} +#endif + int __init init_cpu_ops(int cpu); extern const struct cpu_operations *get_cpu_ops(int cpu); diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h index e9ce68d50ba45..f453f11af781e 100644 --- a/arch/arm64/include/asm/suspend.h +++ b/arch/arm64/include/asm/suspend.h @@ -48,6 +48,38 @@ int swsusp_arch_resume(void); int arch_hibernation_header_save(void *addr, unsigned int max_size); int arch_hibernation_header_restore(void *addr); +/** + * struct arm64_hibernate_layout - What a platform adds to the boot layout check. + * @ex: Ranges that this boot keeps out of the image and that the layout check + * leaves out of the memory it compares, in ascending order. + * @nex: Number of valid entries in @ex. + * @have_base: Whether CPU release addresses are compared as offsets from @base. + * @base: Base address for the release addresses, if @have_base. + */ +struct arm64_hibernate_layout { + u64 ex[2][2]; + int nex; + bool have_base; + u64 base; +}; + +/** + * struct arm64_hibernate_platform - Platform hooks of arm64 hibernation. + * @pfn_nosave: Whether the platform keeps page frame @pfn out of the image. + * @layout: Mix the platform's boot facts into the layout cookie in @h with + * arm64_hibernate_layout_mix(), fill @lay, and return the new hash. The + * cookie is stored with an image and compared before it is restored: the + * boot that resumes an image must give the kernel the layout of the boot + * that wrote it. + */ +struct arm64_hibernate_platform { + bool (*pfn_nosave)(unsigned long pfn); + u64 (*layout)(u64 h, struct arm64_hibernate_layout *lay); +}; + +void arm64_hibernate_set_platform(const struct arm64_hibernate_platform *platform); +u64 arm64_hibernate_layout_mix(u64 h, const void *p, size_t len); + /* Used to resume on the CPU we hibernated on */ int hibernate_resume_nonboot_cpu_disable(void); diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index cac465111055b..263d682d48620 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -65,6 +65,9 @@ obj-$(CONFIG_ARM64_ACPI_PARKING_PROTOCOL) += acpi_parking_protocol.o obj-$(CONFIG_PARAVIRT) += paravirt.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o +ifdef CONFIG_HOTPLUG_CPU +obj-$(CONFIG_HIBERNATION) += smp_spin_table_pen.o +endif obj-$(CONFIG_ELF_CORE) += elfcore.o obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \ cpu-reset.o diff --git a/arch/arm64/kernel/hibernate-asm.S b/arch/arm64/kernel/hibernate-asm.S index 2baefe7a82d36..89fc1db49f618 100644 --- a/arch/arm64/kernel/hibernate-asm.S +++ b/arch/arm64/kernel/hibernate-asm.S @@ -67,19 +67,24 @@ SYM_CODE_START(swsusp_arch_suspend_exit) copy_page x0, x1, x2, x3, x4, x5, x6, x7, x8, x9 add x1, x10, #PAGE_SIZE - /* Clean the copied page to PoU - based on caches_clean_inval_pou() */ + /* + * Clean the copied page to PoC, not only PoU: non-coherent DMA + * masters (the coprocessors of some SoCs) read the restored pages + * from DRAM. Clean+invalidate to PoC also covers what + * caches_clean_inval_pou() did for the I-side. + */ raw_dcache_line_size x2, x3 sub x3, x2, #1 bic x4, x10, x3 2: /* clean D line / unified line */ -alternative_insn "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE + dc civac, x4 add x4, x4, x2 cmp x4, x1 b.lo 2b ldr x19, [x19, #HIBERN_PBE_NEXT] cbnz x19, 1b - dsb ish /* wait for PoU cleaning to finish */ + dsb ish /* wait for PoC cleaning to finish */ /* switch to the restored kernels page tables */ break_before_make_ttbr_switch x25, x21, x6, x8 diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 9398c8d0ecb03..ce251b5b20d0d 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -12,8 +12,11 @@ #define pr_fmt(x) "hibernate: " x #include #include +#include +#include #include #include +#include #include #include @@ -81,6 +84,9 @@ static struct arch_hibernate_hdr { phys_addr_t __hyp_stub_vectors; u64 sleep_cpu_mpidr; + + /* What the boot loader decided for the boot that wrote the image. */ + u64 boot_layout; } resume_hdr; static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i) @@ -89,13 +95,137 @@ static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i) memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version)); } +/** + * arm64_hibernate_layout_mix - Mix bytes into the boot layout cookie. + * @h: Hash so far. + * @p: Bytes to mix. + * @len: Number of bytes. + * + * FNV-1a, for the callback of a platform to mix its boot facts into the cookie + * in the same way as the rest of the layout. + */ +u64 arm64_hibernate_layout_mix(u64 h, const void *p, size_t len) +{ + const u8 *b = p; + + while (len--) { + h ^= *b++; + h *= 0x100000001b3ULL; /* FNV-1a */ + } + return h; +} + +/* + * Mix one memory span less the parts in @ex, ranges that this boot keeps out of + * the image. iBoot's no-map allocations inside aura's fixed layout (the DCP + * oslogs, ANS firmware) move with the boot object and split memblock's regions + * differently on every boot; what lies there never holds an image page. + */ +static u64 hibernate_mix_span(u64 h, u64 b, u64 e, u32 flags, const u64 (*ex)[2], int nex) +{ + u64 size; + + if (b >= e) + return h; + if (nex && e > ex[0][0] && b < ex[0][1]) { + h = hibernate_mix_span(h, b, ex[0][0], flags, ex + 1, nex - 1); + return hibernate_mix_span(h, ex[0][1], e, flags, ex + 1, nex - 1); + } + if (nex) + return hibernate_mix_span(h, b, e, flags, ex + 1, nex - 1); + size = e - b; + h = arm64_hibernate_layout_mix(h, &b, sizeof(b)); + h = arm64_hibernate_layout_mix(h, &size, sizeof(size)); + return arm64_hibernate_layout_mix(h, &flags, sizeof(flags)); +} + +static const struct arm64_hibernate_platform *hibernate_platform __ro_after_init; + +/** + * arm64_hibernate_set_platform - Register the platform's hibernation hooks. + * @platform: Hooks of the platform; they must stay valid. + * + * Only one platform can register, at boot. + */ +void __init arm64_hibernate_set_platform(const struct arm64_hibernate_platform *platform) +{ + if (WARN_ON_ONCE(hibernate_platform)) + return; + hibernate_platform = platform; +} + +static bool hibernate_platform_pfn_nosave(unsigned long pfn) +{ + return hibernate_platform && hibernate_platform->pfn_nosave && + hibernate_platform->pfn_nosave(pfn); +} + +/* + * The image restores every page to the frame it came from, so the boot that + * resumes it must give the kernel what the boot that wrote it did: the same + * RAM, the same CPUs started the same way, and whatever fixed layout the + * platform's boot loader gives the kernel, which the platform's hook adds. A + * boot loader decides them on every boot; the kernel build check above cannot + * see them. + */ +static u64 hibernate_boot_layout(void) +{ + struct arm64_hibernate_layout lay = {}; + u64 h = 0xcbf29ce484222325ULL, rel, hwid; + struct memblock_region *r; + struct device_node *np; + u8 kept; + + if (hibernate_platform && hibernate_platform->layout) + h = hibernate_platform->layout(h, &lay); + + for_each_of_cpu_node(np) { + if (of_property_read_u64(np, "cpu-release-addr", &rel)) + continue; + hwid = of_get_cpu_hwid(np, 0); + kept = hibernate_platform_pfn_nosave(PHYS_PFN(rel)); + if (lay.have_base) + rel -= lay.base; + h = arm64_hibernate_layout_mix(h, &hwid, sizeof(hwid)); + h = arm64_hibernate_layout_mix(h, &rel, sizeof(rel)); + h = arm64_hibernate_layout_mix(h, &kept, sizeof(kept)); + } + + for_each_mem_region(r) + h = hibernate_mix_span(h, r->base, r->base + r->size, r->flags, lay.ex, lay.nex); + return h; +} + +/* + * Most image pages are loaded straight into their original frames through the + * cacheable linear map. Non-coherent DMA masters (on Apple SoCs: the GPU page + * table walker, DCP, every RTKit co-processor, the NVMe queues) read DRAM, and + * a later eviction of a dirty line would overwrite what a device wrote since. + * Clean and invalidate the page to the PoC before the jump. + * + * A frame that is not in the boot kernel's linear map (for example the linear + * alias of the kernel image, which faults when touched) had nothing loaded + * through the linear map: such frames are restored by the restore_pblist copy, + * which cleans each copied page itself. + */ +int arch_hibernate_clean_image_page(unsigned long pfn) +{ + unsigned long va; + + if (!kernel_page_present(pfn_to_page(pfn))) + return 0; + va = (unsigned long)page_address(pfn_to_page(pfn)); + dcache_clean_inval_poc(va, va + PAGE_SIZE); + return 1; +} + int pfn_is_nosave(unsigned long pfn) { unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin); unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1); return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) || - crash_is_nosave(pfn); + crash_is_nosave(pfn) || hibernate_platform_pfn_nosave(pfn); } void notrace save_processor_state(void) @@ -129,8 +259,9 @@ int arch_hibernation_header_save(void *addr, unsigned int max_size) return -ENODEV; } hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu); - pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu, - hdr->sleep_cpu_mpidr); + hdr->boot_layout = hibernate_boot_layout(); + pr_info("Hibernating on CPU %d [mpidr:0x%llx], boot layout %016llx\n", sleep_cpu, + hdr->sleep_cpu_mpidr, hdr->boot_layout); return 0; } @@ -138,6 +269,7 @@ EXPORT_SYMBOL(arch_hibernation_header_save); int arch_hibernation_header_restore(void *addr) { + u64 layout; int ret; struct arch_hibernate_hdr_invariants invariants; struct arch_hibernate_hdr *hdr = addr; @@ -147,6 +279,12 @@ int arch_hibernation_header_restore(void *addr) pr_crit("Hibernate image not generated by this kernel!\n"); return -EINVAL; } + layout = hibernate_boot_layout(); + if (hdr->boot_layout != layout) { + pr_crit("Hibernate image written under another boot layout (%016llx, this boot %016llx): not restored\n", + hdr->boot_layout, layout); + return -EINVAL; + } sleep_cpu = get_logical_index(hdr->sleep_cpu_mpidr); pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu, @@ -337,7 +475,7 @@ int swsusp_arch_suspend(void) unsigned long flags; struct sleep_stack_data state; - if (cpus_are_stuck_in_kernel() || spin_table_cpus_stuck()) { + if (cpus_are_stuck_in_kernel() || spin_table_hibernate_cpus_stuck()) { pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); return -EBUSY; } @@ -386,6 +524,9 @@ int swsusp_arch_suspend(void) sleep_cpu = -EINVAL; __cpu_suspend_exit(); + /* The boot kernel's secondaries are in its pen, not where the image parked them. */ + spin_table_hibernate_restored(); + /* * Just in case the boot kernel did turn the SSBD * mitigation off behind our back, let's set the state @@ -489,10 +630,19 @@ int __nocfi swsusp_arch_resume(void) int hibernate_resume_nonboot_cpu_disable(void) { + int ret; + if (sleep_cpu < 0) { pr_err("Failing to resume from hibernate on an unknown CPU.\n"); return -ENODEV; } - return freeze_secondary_cpus(sleep_cpu); + /* Parking spin-table CPUs wait outside the image while it is restored. */ + ret = spin_table_hibernate_pen_begin(resume_hdr.ttbr1_el1, sleep_cpu); + if (ret) + return ret; + ret = freeze_secondary_cpus(sleep_cpu); + spin_table_hibernate_pen_end(!ret); + + return ret; } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index f6b5decff53da..02badcc245d05 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -1354,14 +1354,29 @@ bool smp_crash_stop_failed(void) } #endif +/* + * The CPUs that have to leave the kernel are the others. A boot CPU whose node + * carries no enable-method (Apple boot objects describe only the secondaries' + * spin table) has no operations of its own: then every other possible CPU must + * have a way out. + */ static bool have_cpu_die(void) { #ifdef CONFIG_HOTPLUG_CPU - int any_cpu = raw_smp_processor_id(); - const struct cpu_operations *ops = get_cpu_ops(any_cpu); + int this_cpu = raw_smp_processor_id(), cpu; + const struct cpu_operations *ops = get_cpu_ops(this_cpu); - if (ops && ops->cpu_die) - return true; + if (ops) + return ops->cpu_die != NULL; + + for_each_possible_cpu(cpu) { + if (cpu == this_cpu) + continue; + ops = get_cpu_ops(cpu); + if (!ops || !ops->cpu_die) + return false; + } + return num_possible_cpus() > 1; #endif return false; } diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 2d7660bd22f80..9d5f26afe4edd 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -5,10 +5,12 @@ * Copyright (C) 2013 ARM Ltd. */ +#include #include #include #include #include +#include #include #include @@ -144,8 +146,10 @@ const struct cpu_operations smp_spin_table_ops = { * masked. From there it goes back through secondary_holding_pen, or, when the * platform suspends the system, it stops and takes its last WFI after the commit. * It is then started again at its reset vector. Stopping and starting a CPU is - * SoC-specific, so a platform driver provides it. Until one has, CPU hotplug - * fails cleanly, as on plain spin-table. + * SoC-specific, so a platform driver provides it (spin_table_set_power_ops()); + * until one has, a suspend that needs the parked CPUs stopped is refused. + * Parking and leaving the park need only cpu_soft_restart(): without + * KEXEC_CORE, CPU hotplug fails cleanly, as on plain spin-table. * * A CPU left awake at the commit can hang the machine, and the commit can't be * undone. So each parked CPU first answers a hold, while the suspend can still @@ -163,6 +167,198 @@ static bool spin_table_stop_released; #define SPIN_TABLE_WAIT_US 100000 +#if defined(CONFIG_HIBERNATION) && defined(CONFIG_KEXEC_CORE) +/* + * Resuming from hibernation, the boot kernel takes its secondaries offline and + * then overwrites every page the image saved, the park and its cpumasks + * included. A CPU parked there would run on in memory that changes under it, + * and its translation tables with it. On that path a CPU leaving the kernel + * waits instead in a pen the image doesn't contain: the code and a slot per + * CPU in .data..nosave, run with the MMU off. arm64 resumes only an image of + * the kernel build that is running; with the boot kernel loaded at the image + * kernel's physical address (checked when the resume starts) the pen is at the + * same place in both, and the restored kernel reads what the boot kernel left + * there. It starts each CPU from the pen through secondary_holding_pen, as at + * boot. Nothing is powered off, so this needs no platform power ops. + */ +#define SPIN_TABLE_PEN_MAGIC 0x314e455034535053ULL /* "SPS4PEN1" */ +#define SPIN_TABLE_PEN_HERE 0x45524548ULL /* "HERE", from the pen */ + +struct spin_table_pen_slot { + u64 release; /* where the CPU goes; 0 holds it */ + u64 here; /* the CPU reached the pen */ + u64 mpidr; +} __aligned(ARCH_DMA_MINALIGN); + +static struct { + u64 magic; + u64 self; /* this record's address in the kernel that wrote it */ + unsigned long penned[BITS_TO_LONGS(NR_CPUS)]; + struct spin_table_pen_slot slot[NR_CPUS]; +} spin_table_pen __nosavedata __aligned(ARCH_DMA_MINALIGN); + +extern char spin_table_hibernate_pen[]; +extern const struct cpu_operations smp_rvbar_spin_table_ops; +static struct cpumask spin_table_cpus_penned; +static bool spin_table_pen_requested; + +static void spin_table_pen_clean(void *start, size_t size) +{ + dcache_clean_inval_poc((unsigned long)start, (unsigned long)start + size); +} + +/* + * Resuming from hibernation, before the boot kernel takes its secondaries offline. + * The restored kernel finds the pen only where it is in its own image, which holds + * when the boot kernel is loaded at the image kernel's physical address. arm64 can + * restore from a boot kernel placed elsewhere (@image_ttbr1, the image's + * swapper_pg_dir, is used for the jump), but there the pen's frames are image + * frames: the restore would overwrite them while the penned CPUs run from them with + * the MMU off. Refuse the resume then, as long as a CPU other than @primary (the + * one that stays online) would need the pen. + */ +int spin_table_hibernate_pen_begin(phys_addr_t image_ttbr1, int primary) +{ + phys_addr_t here = __pa_symbol(swapper_pg_dir); + int cpu; + + /* No record of an earlier, abandoned resume may reach the restored kernel. */ + spin_table_pen.magic = 0; + spin_table_pen_clean(&spin_table_pen, sizeof(spin_table_pen.magic)); + + if (image_ttbr1 != here) { + for_each_online_cpu(cpu) { + if (cpu == primary || get_cpu_ops(cpu) != &smp_rvbar_spin_table_ops) + continue; + pr_err("hibernation: the image's kernel ran at another physical address (swapper_pg_dir %pa, here %pa): spin-table CPUs cannot wait in this kernel's pen, resume refused\n", + &image_ttbr1, &here); + return -EINVAL; + } + return 0; + } + WRITE_ONCE(spin_table_pen_requested, true); + return 0; +} + +/* + * After they went offline. With all of them frozen, the restored kernel finds the + * record; otherwise the resume is abandoned and they go back online from the pen. + */ +void spin_table_hibernate_pen_end(bool frozen) +{ + unsigned int cpu; + + WRITE_ONCE(spin_table_pen_requested, false); + if (!frozen || cpumask_empty(&spin_table_cpus_penned)) + return; + + bitmap_zero(spin_table_pen.penned, NR_CPUS); + for_each_cpu(cpu, &spin_table_cpus_penned) + __set_bit(cpu, spin_table_pen.penned); + spin_table_pen.self = __pa_symbol(&spin_table_pen); + spin_table_pen.magic = SPIN_TABLE_PEN_MAGIC; + spin_table_pen_clean(&spin_table_pen, offsetof(typeof(spin_table_pen), slot)); + pr_info("hibernation: %u CPUs wait outside the image\n", + cpumask_weight(&spin_table_cpus_penned)); +} + +/* + * In the restored kernel, right after the image took over: the CPUs the image parked + * before its snapshot are in the boot kernel's pen now. + */ +void spin_table_hibernate_restored(void) +{ + unsigned int cpu, found = 0; + + if (spin_table_pen.magic != SPIN_TABLE_PEN_MAGIC) + return; + spin_table_pen.magic = 0; + spin_table_pen_clean(&spin_table_pen, sizeof(spin_table_pen.magic)); + if (spin_table_pen.self != __pa_symbol(&spin_table_pen)) { + pr_err("hibernation: the boot kernel's CPU pen was at %#llx, not here: its CPUs stay offline\n", + spin_table_pen.self); + return; + } + + cpumask_clear(&spin_table_cpus_penned); + for_each_present_cpu(cpu) { + if (!test_bit(cpu, spin_table_pen.penned)) + continue; + if (spin_table_pen.slot[cpu].mpidr != cpu_logical_map(cpu)) { + pr_err("hibernation: CPU%u: the pen holds MPIDR %#llx, not %#llx\n", cpu, + spin_table_pen.slot[cpu].mpidr, cpu_logical_map(cpu)); + continue; + } + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + cpumask_clear_cpu(cpu, &spin_table_cpus_held); + cpumask_clear_cpu(cpu, &spin_table_cpus_unpark); + cpumask_clear_cpu(cpu, &spin_table_cpus_off); + cpumask_set_cpu(cpu, &spin_table_cpus_penned); + found++; + } + pr_info("hibernation: %u CPUs start from the boot kernel's pen\n", found); +} + +/* Leaves the kernel for the pen; the boot CPU waits for it to arrive (cpu_kill). */ +static void __noreturn smp_rvbar_spin_table_pen(unsigned int cpu, u64 mpidr) +{ + struct spin_table_pen_slot *slot = &spin_table_pen.slot[cpu]; + typeof(cpu_soft_restart) *restart; + + slot->release = 0; + slot->here = 0; + slot->mpidr = mpidr; + spin_table_pen_clean(slot, sizeof(*slot)); + cpumask_set_cpu(cpu, &spin_table_cpus_penned); + + cpu_install_idmap(); + restart = (void *)__pa_symbol(cpu_soft_restart); + restart(0, __pa_symbol(spin_table_hibernate_pen), __pa_symbol(slot), 0, 0); + unreachable(); +} + +static bool spin_table_pen_arrived(unsigned int cpu) +{ + struct spin_table_pen_slot *slot = &spin_table_pen.slot[cpu]; + + /* Written with the MMU off: drop any copy of the line before reading. */ + dcache_inval_poc((unsigned long)slot, (unsigned long)(slot + 1)); + return READ_ONCE(slot->here) == SPIN_TABLE_PEN_HERE; +} + +/* Starts a CPU waiting in the pen; it goes on to secondary_holding_pen. */ +static void spin_table_pen_release(unsigned int cpu) +{ + struct spin_table_pen_slot *slot = &spin_table_pen.slot[cpu]; + + WRITE_ONCE(slot->release, __pa_symbol(secondary_holding_pen)); + spin_table_pen_clean(slot, sizeof(*slot)); +} +#else +static const bool spin_table_pen_requested; +static struct cpumask spin_table_cpus_penned; + +static void smp_rvbar_spin_table_pen(unsigned int cpu, u64 mpidr) +{ +} + +static bool spin_table_pen_arrived(unsigned int cpu) +{ + return false; +} + +static void spin_table_pen_release(unsigned int cpu) +{ +} + +#ifdef CONFIG_HIBERNATION +/* Without cpu_soft_restart() a CPU cannot leave the kernel: nothing to keep. */ +int spin_table_hibernate_pen_begin(phys_addr_t image_ttbr1, int primary) { return 0; } +void spin_table_hibernate_pen_end(bool frozen) {} +void spin_table_hibernate_restored(void) {} +#endif +#endif + int spin_table_set_power_ops(const struct spin_table_power_ops *ops) { return cmpxchg(&spin_table_power, NULL, ops) ? -EBUSY : 0; @@ -208,6 +404,9 @@ int spin_table_hold_parked(void) { int us; + if (!READ_ONCE(spin_table_power)) + return -EOPNOTSUPP; + WRITE_ONCE(spin_table_hold_requested, true); dsb(ish); sev(); @@ -278,7 +477,9 @@ static int smp_rvbar_spin_table_cpu_boot(unsigned int cpu) /* The CPU may run straight into the pen: release it before it starts. */ write_pen_release(mpidr); - if (cpumask_test_and_clear_cpu(cpu, &spin_table_cpus_off)) { + if (cpumask_test_and_clear_cpu(cpu, &spin_table_cpus_penned)) { + spin_table_pen_release(cpu); + } else if (cpumask_test_and_clear_cpu(cpu, &spin_table_cpus_off)) { ret = spin_table_power->cpu_on(mpidr); if (ret) { cpumask_set_cpu(cpu, &spin_table_cpus_off); @@ -301,10 +502,11 @@ static bool smp_rvbar_spin_table_cpu_can_disable(unsigned int cpu) static int smp_rvbar_spin_table_cpu_disable(unsigned int cpu) { - if (!IS_ENABLED(CONFIG_KEXEC_CORE)) - return -EOPNOTSUPP; - - return READ_ONCE(spin_table_power) ? 0 : -EOPNOTSUPP; + /* + * A parked CPU comes back through cpu_soft_restart(). Only stopping it for a + * suspend needs the platform's power ops (spin_table_hold_parked()). + */ + return IS_ENABLED(CONFIG_KEXEC_CORE) ? 0 : -EOPNOTSUPP; } #ifdef CONFIG_KEXEC_CORE @@ -350,11 +552,14 @@ static void smp_rvbar_spin_table_cpu_die(unsigned int cpu) u64 mpidr = cpu_logical_map(cpu); /* The early-boot and crash paths get here without hotplug support. */ - if (!READ_ONCE(spin_table_power) || !IS_ENABLED(CONFIG_KEXEC_CORE)) { + if (!IS_ENABLED(CONFIG_KEXEC_CORE)) { while (1) wfe(); } + if (READ_ONCE(spin_table_pen_requested)) + smp_rvbar_spin_table_pen(cpu, mpidr); + cpumask_set_cpu(cpu, &spin_table_cpus_parked); while (!cpumask_test_cpu(cpu, &spin_table_cpus_unpark)) { @@ -373,12 +578,11 @@ static int smp_rvbar_spin_table_cpu_kill(unsigned int cpu) { int us; - if (!READ_ONCE(spin_table_power)) - return 0; - for (us = 0; us < SPIN_TABLE_WAIT_US; us += 10) { if (cpumask_test_cpu(cpu, &spin_table_cpus_parked)) return 0; + if (cpumask_test_cpu(cpu, &spin_table_cpus_penned) && spin_table_pen_arrived(cpu)) + return 0; udelay(10); } @@ -409,4 +613,17 @@ bool spin_table_cpus_stuck(void) return true; return false; } + +#ifdef CONFIG_HIBERNATION +/* + * For a hibernation that only holds without the pen. With it the kernel that + * resumes an image takes its offlined CPUs out of the memory the restore + * overwrites before it starts (spin_table_hibernate_pen_begin()), and the + * restored kernel starts them from there. + */ +bool spin_table_hibernate_cpus_stuck(void) +{ + return !IS_ENABLED(CONFIG_KEXEC_CORE) && spin_table_cpus_stuck(); +} +#endif #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/arm64/kernel/smp_spin_table_pen.S b/arch/arm64/kernel/smp_spin_table_pen.S new file mode 100644 index 0000000000000..4610bc09ad002 --- /dev/null +++ b/arch/arm64/kernel/smp_spin_table_pen.S @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Where a parking spin-table CPU waits while the kernel resuming from + * hibernation overwrites memory with the image (smp_spin_table.c). + */ + +#include +#include + +/* + * x0: the CPU's struct spin_table_pen_slot: the entry address at offset 0, + * written by the kernel that starts the CPU again, and the arrival word at 8, + * written here. + * + * Entered from cpu_soft_restart() with the MMU and caches off and interrupts + * masked, so nothing here depends on the kernel's page tables or on cache + * maintenance by this CPU. The code and the slots are in .data..nosave: the + * image never contains them, so the restore leaves both as the boot kernel + * wrote them, at the same physical address in the restored kernel. + */ + .pushsection ".data..nosave", "aw" + .balign 64 +SYM_CODE_START(spin_table_hibernate_pen) + movz x1, #0x4548 + movk x1, #0x4552, lsl #16 // "HERE" + str x1, [x0, #8] + dsb sy +1: ldr x1, [x0] + cbnz x1, 2f + wfe + b 1b +2: br x1 +SYM_CODE_END(spin_table_hibernate_pen) + .popsection diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 6ac3321f4c575..0773f1c1209e8 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -1436,6 +1437,58 @@ static void kvm_timer_handle_errata(void) } } +/* + * A guest's counters are the physical counter minus an offset that is fixed + * when the VM is made. The physical counter can start again in a sleep: it + * always does across a resume from disk, which is a cold boot. The guest's + * counters then fall back by what the counter lost, and the guest's clock + * stands still until the counter has caught up, which takes as long as the + * host had been up. Move the offsets by the same amount instead: the + * guest's counters go on from the value they had when the system went down, + * as for a VM that was stopped that long. A counter that keeps running + * through the sleep is left alone, and the guest sees the sleep as time that + * passed. + * + * Syscore: one CPU, interrupts off, every task frozen, so no vCPU is loaded + * and nothing changes vm_list. + */ +static u64 kvm_timer_sleep_cnt; + +static int kvm_timer_syscore_suspend(void *data) +{ + kvm_timer_sleep_cnt = kvm_phys_timer_read(); + return 0; +} + +static void kvm_timer_syscore_resume(void *data) +{ + u64 now = kvm_phys_timer_read(); + unsigned int vms = 0; + struct kvm *kvm; + u64 lost; + + if (now >= kvm_timer_sleep_cnt) + return; + lost = kvm_timer_sleep_cnt - now; + list_for_each_entry(kvm, &vm_list, vm_list) { + kvm->arch.timer_data.voffset -= lost; + kvm->arch.timer_data.poffset -= lost; + vms++; + } + if (vms) + kvm_info("the counter started again in the sleep (%llu -> %llu): the counters of %u VM(s) kept\n", + kvm_timer_sleep_cnt, now, vms); +} + +static const struct syscore_ops kvm_timer_syscore_ops = { + .suspend = kvm_timer_syscore_suspend, + .resume = kvm_timer_syscore_resume, +}; + +static struct syscore kvm_timer_syscore = { + .ops = &kvm_timer_syscore_ops, +}; + int __init kvm_timer_hyp_init(bool has_gic) { struct arch_timer_kvm_info *info; @@ -1505,6 +1558,7 @@ int __init kvm_timer_hyp_init(bool has_gic) } kvm_timer_handle_errata(); + register_syscore(&kvm_timer_syscore); return 0; out_free_ptimer_irq: diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 02cc8fccbc27c..7c545e355df5d 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -85,6 +85,8 @@ enum bcm4377_chip { #define BCM4377_BAR0_HOST_WINDOW_SIZE 0x598 #define BCM4377_BAR2_BOOTSTAGE 0x200454 +/* The firmware has booted. */ +#define BCM4377_BOOTSTAGE_RUNNING 2 #define BCM4377_BAR2_FW_LO 0x200478 #define BCM4377_BAR2_FW_HI 0x20047c @@ -96,6 +98,8 @@ enum bcm4377_chip { #define BCM4388_BAR2_CONTEXT_ADDR_HI 0x200490 #define BCM4377_BAR2_RTI_STATUS 0x20045c +/* The firmware has taken the context and serves the rings. */ +#define BCM4377_RTI_STATUS_RUNNING 2 #define BCM4377_BAR2_RTI_WINDOW_LO 0x200494 #define BCM4377_BAR2_RTI_WINDOW_HI 0x200498 #define BCM4377_BAR2_RTI_WINDOW_SIZE 0x20049c @@ -619,7 +623,12 @@ struct bcm4377_data { bool resume_powered; bool resume_guarded; bool resume_recovering; + /* Back from a system sleep without its firmware: nothing is sent until the reset. */ + bool resume_cold; + /* The HCI core is still suspended: it resumes after the open that boots the firmware. */ + bool hci_resume_owed; struct work_struct resume_work; + struct work_struct hci_resume_work; bdaddr_t bdaddr; wait_queue_head_t event; @@ -1477,6 +1486,7 @@ static int bcm4377_pci_reset(struct bcm4377_data *bcm4377) } bcm4377->needs_reset = false; + WRITE_ONCE(bcm4377->resume_cold, false); dev_info(&bcm4377->pdev->dev, "hardware reset recovery completed successfully\n"); return 0; @@ -1592,10 +1602,46 @@ static void bcm4377_setup_retry_work(struct work_struct *work) queue_work(hdev->req_workqueue, &hdev->power_on); } +/* + * Boards whose transport is restarted in place, by a close and an open that resets the + * function: when a command gets no answer after a resume, and when a sleep left the + * function without its firmware. + */ +static const struct bcm4377_resume_board { + unsigned int id; + const char *machine; + /* + * A suspend to RAM takes the firmware. The core then resumes after the reset, from + * this driver, which makes its system sleep callbacks the only ones that suspend + * and resume the core. + */ + bool cold_sleep; +} bcm4377_resume_boards[] = { + { .id = 0x4378, .machine = "apple,j493" }, + { .id = 0x4388, .machine = "apple,j616s", .cold_sleep = true }, + { } +}; + +static const struct bcm4377_resume_board *bcm4377_resume_board(struct bcm4377_data *bcm4377) +{ + const struct bcm4377_resume_board *board; + + for (board = bcm4377_resume_boards; board->machine; board++) + if (bcm4377->hw->id == board->id && of_machine_is_compatible(board->machine)) + return board; + return NULL; +} + static bool bcm4377_resume_recovery_supported(struct bcm4377_data *bcm4377) { - return bcm4377->hw->id == 0x4378 && - of_machine_is_compatible("apple,j493"); + return bcm4377_resume_board(bcm4377); +} + +static bool bcm4377_resume_cold_sleep(struct bcm4377_data *bcm4377) +{ + const struct bcm4377_resume_board *board = bcm4377_resume_board(bcm4377); + + return board && board->cold_sleep; } static bool bcm4377_resume_recovery_allowed(struct hci_dev *hdev) @@ -1642,8 +1688,12 @@ static int bcm4377_resume_recovery_sync(struct hci_dev *hdev, void *data) if (!allowed) return 0; - dev_warn(&bcm4377->pdev->dev, - "recovering transport once after resume command timeout\n"); + if (READ_ONCE(bcm4377->resume_cold)) + dev_info(&bcm4377->pdev->dev, + "restarting the transport: the sleep left no firmware running\n"); + else + dev_warn(&bcm4377->pdev->dev, + "recovering transport once after resume command timeout\n"); ret = hci_reset_dev_sync(hdev); spin_lock_irqsave(&bcm4377->resume_lock, flags); bcm4377->resume_recovering = false; @@ -1688,6 +1738,45 @@ static void bcm4377_resume_recovery_work(struct work_struct *work) bcm4377_resume_recovery_destroy(bcm4377->hdev, epoch, ret); } +/* Resume the HCI core. It stays suspended, and owed, while the controller is not powered. */ +static int bcm4377_hci_resume(struct bcm4377_data *bcm4377) +{ + struct hci_dev *hdev = bcm4377->hdev; + int ret = hci_resume_dev(hdev); + + if (!READ_ONCE(hdev->suspended)) + WRITE_ONCE(bcm4377->hci_resume_owed, false); + return ret; +} + +/* + * A function that came back from a sleep without its firmware answers none of the commands + * the core's resume sends. The core stays suspended until an open has reset the function + * and booted the firmware, and resumes here, after that open. + */ +static void bcm4377_hci_resume_work(struct work_struct *work) +{ + struct bcm4377_data *bcm4377 = container_of(work, struct bcm4377_data, + hci_resume_work); + struct hci_dev *hdev = bcm4377->hdev; + + /* The open that queued this holds the request lock until the controller is up. */ + hci_req_sync_lock(hdev); + hci_req_sync_unlock(hdev); + if (READ_ONCE(bcm4377->hci_resume_owed)) + bcm4377_hci_resume(bcm4377); +} + +static void bcm4377_queue_hci_resume(struct bcm4377_data *bcm4377) +{ + unsigned long flags; + + spin_lock_irqsave(&bcm4377->resume_lock, flags); + if (READ_ONCE(bcm4377->hci_resume_owed) && !bcm4377->resume_blocked) + queue_work(system_freezable_wq, &bcm4377->hci_resume_work); + spin_unlock_irqrestore(&bcm4377->resume_lock, flags); +} + static void bcm4377_hci_reset(struct hci_dev *hdev); static void bcm4377_hci_timeout(struct hci_dev *hdev) @@ -1735,6 +1824,8 @@ static void bcm4377_resume_recovery_stop(struct bcm4377_data *bcm4377) NULL, NULL); /* A callback already removed from the queue must retire before PCI quiesces. */ flush_work(&bcm4377->hdev->cmd_sync_work); + /* An open that ended ahead of this fence may have queued the core's resume. */ + cancel_work_sync(&bcm4377->hci_resume_work); } static int bcm4377_hci_open(struct hci_dev *hdev) @@ -1813,8 +1904,10 @@ static int bcm4377_hci_close(struct hci_dev *hdev) } if (bcm4377->needs_reset) { - dev_warn(&bcm4377->pdev->dev, - "transport needs hardware reset on next open\n"); + /* Expected of a function a sleep left cold; its restart has said so. */ + if (!READ_ONCE(bcm4377->resume_cold)) + dev_warn(&bcm4377->pdev->dev, + "transport needs hardware reset on next open\n"); bcm4377->acl_d2h_ring.enabled = false; bcm4377->acl_h2d_ring.enabled = false; bcm4377->sco_d2h_ring.enabled = false; @@ -1948,7 +2041,7 @@ static int bcm4377_hci_post_init(struct hci_dev *hdev) int ret; if (!bcm4377->cal_needed) - return 0; + goto out; if (!bcm4377->cal_loaded) { dev_info(&bcm4377->pdev->dev, @@ -1982,6 +2075,9 @@ static int bcm4377_hci_post_init(struct hci_dev *hdev) } bcm4377->cal_needed = false; +out: + /* The firmware answers: a core that a cold resume left suspended can resume. */ + bcm4377_queue_hci_resume(bcm4377); return 0; } @@ -1991,6 +2087,12 @@ static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb) struct bcm4377_transfer_ring *ring; int ret; + /* No firmware behind the rings: fail at once, a command must not wait out its timeout. */ + if (READ_ONCE(bcm4377->resume_cold)) { + hdev->stat.err_tx++; + return -ENODEV; + } + switch (hci_skb_pkt_type(skb)) { case HCI_COMMAND_PKT: hdev->stat.cmd_tx++; @@ -2964,6 +3066,7 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) INIT_DELAYED_WORK(&bcm4377->setup_retry_work, bcm4377_setup_retry_work); spin_lock_init(&bcm4377->resume_lock); INIT_WORK(&bcm4377->resume_work, bcm4377_resume_recovery_work); + INIT_WORK(&bcm4377->hci_resume_work, bcm4377_hci_resume_work); ret = bcm4377_prepare_rings(bcm4377); if (ret) @@ -3069,6 +3172,12 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdev->post_init = bcm4377_hci_post_init; if (bcm4377_resume_recovery_supported(bcm4377)) hdev->reset = bcm4377_hci_timeout; + /* + * The core's notifier would resume the controller as soon as tasks are thawed, when + * the function a sleep left cold is not reset yet. bcm4377_resume() does it instead. + */ + if (bcm4377_resume_cold_sleep(bcm4377)) + hci_set_quirk(hdev, HCI_QUIRK_NO_SUSPEND_NOTIFIER); if (bcm4377->hw->broken_mws_transport_config) hci_set_quirk(hdev, HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG); @@ -3178,11 +3287,36 @@ static int bcm4377_suspend(struct device *dev) return ret; } +/* + * A system sleep that takes the chip's power or resets it leaves the function's private + * windows at their reset values and no firmware: a BCM4388 then reads window 1 as + * 0x19328000 and, once the windows are back, boot stage 0. Put the windows back and say + * whether a firmware still runs behind them. + */ +static bool bcm4377_resume_cold(struct bcm4377_data *bcm4377) +{ + u32 win = bcm4377->hw->bar0_window1, bootstage, rti_status; + + pci_read_config_dword(bcm4377->pdev, BCM4377_PCIECFG_BAR0_WINDOW1, &win); + if (win == bcm4377->hw->bar0_window1) + return false; + if (bcm4377_init_cfg(bcm4377)) + return true; + bootstage = ioread32(bcm4377->bar2 + bcm4377->hw->bar2_offset + BCM4377_BAR2_BOOTSTAGE); + rti_status = ioread32(bcm4377->bar2 + bcm4377->hw->bar2_offset + BCM4377_BAR2_RTI_STATUS); + dev_dbg(&bcm4377->pdev->dev, + "the sleep reset the function (window %#x): boot stage %u, RTI status %u\n", + win, bootstage, rti_status); + return bootstage != BCM4377_BOOTSTAGE_RUNNING || + rti_status != BCM4377_RTI_STATUS_RUNNING; +} + static int bcm4377_resume(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev); unsigned long flags; + bool cold; int ret; ret = apple_radio_sleep_endpoint_resume(dev); @@ -3194,6 +3328,16 @@ static int bcm4377_resume(struct device *dev) bcm4377->wake_armed = false; } + /* A function an earlier sleep left cold stays cold until a reset boots it. */ + cold = bcm4377_resume_recovery_supported(bcm4377) && + (bcm4377_resume_cold(bcm4377) || READ_ONCE(bcm4377->resume_cold)); + if (cold) { + /* No firmware to wake: the next open resets the function and boots it. */ + bcm4377->quiesced = false; + bcm4377->needs_reset = true; + WRITE_ONCE(bcm4377->resume_cold, true); + } + if (bcm4377->quiesced) { iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE, bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL); @@ -3211,13 +3355,36 @@ static int bcm4377_resume(struct device *dev) } bcm4377->resume_powered = false; - ret = hci_resume_dev(bcm4377->hdev); - apple_radio_sleep_endpoint_resumed(dev, ret); - if (!apple_radio_sleep_owned(dev)) { + /* The radio-sleep owner's endpoint resumes as it suspended: whole, and reports it. */ + if (apple_radio_sleep_owned(dev)) { + ret = bcm4377_hci_resume(bcm4377); + apple_radio_sleep_endpoint_resumed(dev, ret); + return ret; + } + if (cold) { + /* What a command timeout would start, without the timeout. */ + spin_lock_irqsave(&bcm4377->resume_lock, flags); + if (bcm4377->resume_armed) { + bcm4377->resume_armed = false; + bcm4377->resume_pending = bcm4377->resume_epoch; + /* Nothing answers before that reset: the core resumes after the open. */ + WRITE_ONCE(bcm4377->hci_resume_owed, true); + queue_work(system_freezable_wq, &bcm4377->resume_work); + } + spin_unlock_irqrestore(&bcm4377->resume_lock, flags); + /* No reset queued: the controller is down or the core's resume fails at once. */ + if (!READ_ONCE(bcm4377->hci_resume_owed)) + bcm4377_hci_resume(bcm4377); WRITE_ONCE(bcm4377->setup_retry_blocked, false); - if (!ret) - bcm4377_queue_setup_retry(bcm4377); + /* A setup retry the suspend held back needs no firmware: its open resets. */ + bcm4377_queue_setup_retry(bcm4377); + return 0; } + + ret = bcm4377_hci_resume(bcm4377); + WRITE_ONCE(bcm4377->setup_retry_blocked, false); + if (!ret) + bcm4377_queue_setup_retry(bcm4377); return ret; } diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 379fa031ce599..c989aa9eb3f97 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -1059,6 +1059,14 @@ static void apple_fast_die_sample(struct apple_fast_die *fd) fd->waiting = false; } else { limit = fd->fallback; + /* + * The PMP stopped after it had run (a resume from disk restarts it): + * the wait, and its warning, start over from now. + */ + if (!fd->waiting && fd->report) { + fd->start = jiffies; + fd->warned = false; + } fd->waiting = true; if (!fd->warned && time_after(jiffies, fd->start + APPLE_FAST_DIE_WARN_S * HZ)) { diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index 330bb7fa6b753..a27b50591a610 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -645,6 +645,7 @@ static const struct sio_hw_data sio_hw_t6030 = { * the 5 of the host's mailbox call timeout is not part of the record. */ static const struct sio_hw_data sio_hw_t6041 = { + .stop_at_commit = true, .protocol = 10, .fifo_threshold = 0x600, .cfg_word4 = 0x800, @@ -1838,6 +1839,20 @@ static int sio_send_v9_shmem(struct sio_data *sio, return 0; } +/* Publish the shared memory to the firmware; also used after a cold restart. */ +static int sio_send_shmem_params(struct sio_data *sio) +{ + int err; + + if (sio->protocol == 9) + return sio_send_v9_shmem(sio, sio_set_param); + + err = sio_set_param(sio, 1, sio->shmem_iova >> 12); + if (err) + return err; + return sio_set_param(sio, 2, SIO_SHMEM_SIZE); +} + static int sio_alloc_shmem(struct sio_data *sio) { const struct sio_dma_arrays *arrays = sio->hw->arrays; @@ -1879,13 +1894,8 @@ static int sio_alloc_shmem(struct sio_data *sio) err = devm_add_action_or_reset(sio->dev, sio_free_v9_shmem, sio); if (err) return err; - return sio_send_v9_shmem(sio, sio_set_param); } - - err = sio_set_param(sio, 1, sio->shmem_iova >> 12); - if (err) - return err; - return sio_set_param(sio, 2, SIO_SHMEM_SIZE); + return sio_send_shmem_params(sio); free_shmem: /* No address has been published yet. */ @@ -2141,9 +2151,15 @@ static bool sio_chan_busy(struct sio_chan *siochan) static __maybe_unused int sio_suspend_late(struct device *dev) { struct sio_data *sio = dev_get_drvdata(dev); + bool dry; int i, ret; - if (!pm_suspend_via_firmware() || sio->asleep) + /* + * A dry run keeps the power, and an idled firmware doesn't wake with it kept. One whose + * stop is the commit's is ready there as for a real sleep: a dry run stops nothing. + */ + dry = !apple_sleep_power_loss() && sio->commit_registered && pm_suspend_via_firmware(); + if ((!apple_sleep_power_loss() && !dry) || sio->asleep) return 0; for (i = 0; i < sio->nchannels; i++) { @@ -2153,7 +2169,7 @@ static __maybe_unused int sio_suspend_late(struct device *dev) } } - if (sio->commit_registered && apple_sleep_commit_stops_coprocs()) { + if (sio->commit_registered && (dry || apple_sleep_commit_stops_coprocs())) { apple_sleep_coproc_set_ready(dev, true); sio->commit_used = true; sio->asleep = true; @@ -2178,6 +2194,70 @@ static __maybe_unused int sio_suspend_late(struct device *dev) return 0; } +/* + * Bring a freshly loaded, halted SIO up again under the existing host state: + * the firmware-facing part of probe, reusing the shared memory, the DMA + * channels and their clients. + */ +static int sio_restart(struct sio_data *sio) +{ + u32 protocol = sio->protocol; + int err, i; + + /* Nothing can answer the calls that were in flight. */ + for (i = 1; i < SIO_NTAGS; i++) { + if (test_and_clear_bit(i, sio->tags.allocated)) + dev_warn(sio->dev, "dropping stale call tag %d\n", i); + reinit_completion(&sio->tags.completions[i]); + } + + err = apple_rtkit_reinit(sio->rtk); + if (err) { + dev_err(sio->dev, "restart: RTKit reinit: %d\n", err); + return err; + } + + writel(CPU_CONTROL_RUN, sio->base + REG_CPU_CONTROL); + + err = apple_rtkit_boot(sio->rtk); + if (err) { + dev_err(sio->dev, "restart: SIO did not boot: %d\n", err); + return err; + } + + err = apple_rtkit_start_ep(sio->rtk, EP_SIO); + if (err) { + dev_err(sio->dev, "restart: starting SIO endpoint: %d\n", err); + return err; + } + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_START)); + if (err < 1) { + err = err ? err : -ENXIO; + dev_err(sio->dev, "restart: starting SIO service: %d\n", err); + return err; + } + if (sio->protocol != protocol) { + dev_err(sio->dev, "restart: protocol changed from v%u to v%u\n", + protocol, sio->protocol); + return -EPROTO; + } + + err = sio_send_dt_params(sio); + if (err < 0) { + dev_err(sio->dev, "restart: boot-up parameters: %d\n", err); + return err; + } + + err = sio_send_shmem_params(sio); + if (err < 0) { + dev_err(sio->dev, "restart: shared memory: %d\n", err); + return err; + } + + return 0; +} + static __maybe_unused int sio_resume_early(struct device *dev) { struct sio_data *sio = dev_get_drvdata(dev); @@ -2232,8 +2312,40 @@ static __maybe_unused int sio_resume_early(struct device *dev) return 0; } +static __maybe_unused int sio_restore(struct device *dev) +{ + struct sio_data *sio = dev_get_drvdata(dev); + int err; + + /* + * The resume boot loaded the SIO firmware again and left the CPU + * halted (this driver is not part of the resume initramfs), while the + * restored session still believes the old instance runs. Start the + * new instance; DMA clients set their channels up again when they + * reprepare their streams. + */ + if (readl(sio->base + REG_CPU_CONTROL) & CPU_CONTROL_RUN) { + dev_dbg(dev, "resume from disk: SIO still running, session kept\n"); + } else { + err = sio_restart(sio); + if (!err) + dev_dbg(dev, "resume from disk: SIO restarted (protocol v%u)\n", + sio->protocol); + else + dev_err(dev, "resume from disk: SIO restart failed: %d\n", err); + } + + return pm_runtime_force_resume(dev); +} + +/* A firmware-driven suspend stops the firmware late; a resume from disk restarts it. */ static const struct dev_pm_ops sio_pm_ops = { - SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) + .suspend = pm_sleep_ptr(pm_runtime_force_suspend), + .resume = pm_sleep_ptr(pm_runtime_force_resume), + .freeze = pm_sleep_ptr(pm_runtime_force_suspend), + .thaw = pm_sleep_ptr(pm_runtime_force_resume), + .poweroff = pm_sleep_ptr(pm_runtime_force_suspend), + .restore = pm_sleep_ptr(sio_restore), LATE_SYSTEM_SLEEP_PM_OPS(sio_suspend_late, sio_resume_early) RUNTIME_PM_OPS(sio_suspend, sio_resume, NULL) }; diff --git a/drivers/gpu/drm/asahi/g17/sleep.c b/drivers/gpu/drm/asahi/g17/sleep.c index 2aebc75d2188f..e3e0d9c3c654f 100644 --- a/drivers/gpu/drm/asahi/g17/sleep.c +++ b/drivers/gpu/drm/asahi/g17/sleep.c @@ -322,7 +322,7 @@ struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, if (c->request) apple_pmp_report_entry_commit_owned(c->request, true); - ret = apple_sleep_coproc_register_first(dev, asahi_g17_commit_stop, c); + ret = apple_sleep_coproc_register_first(dev, APPLE_SLEEP_FIRST_GPU, asahi_g17_commit_stop, c); if (ret) goto err; apple_sleep_coproc_set_ready(dev, true); @@ -369,7 +369,7 @@ struct asahi_g17_commit *asahi_g16_commit_prepare(struct device *dev, struct app if (!c->cpu[0] || !c->ps) goto err; - ret = apple_sleep_coproc_register_first(dev, asahi_g17_commit_stop, c); + ret = apple_sleep_coproc_register_first(dev, APPLE_SLEEP_FIRST_GPU, asahi_g17_commit_stop, c); if (ret) goto err; apple_sleep_coproc_set_ready(dev, true); diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 89d24f77081b0..a45c2af4933bb 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -208,6 +209,11 @@ struct dockchannel_hid { bool suspended; /* The firmware was told of a sleep and has not been told of the wake */ bool host_asleep; + /* Boot kernel of a resume from disk: input handed to the image. */ + bool hib_quiesced; + u32 hib_off_mask; + /* After taking the ring over: one resync to the producer allowed. */ + bool ring_resync; /* Held from probe on when the multi-touch interface uses power method 2 */ struct gpio_desc *afe_reset; @@ -743,6 +749,50 @@ static int dchid_request_gpio(struct dchid_iface *iface) return 0; } +/* Upload the interface firmware, then reset the interface to run it. */ +static int dchid_load_firmware(struct dchid_iface *iface, void *fw, size_t size) +{ + int ret; + + /* Send firmware to the device */ + dev_dbg(iface->dchid->dev, "Sending firmware for %s\n", iface->name); + /* Retained MTP must see AFE reset throughout firmware upload. */ + if (iface->dchid->inherited_mtp && + iface->power_method == DCHID_POWER_METHOD_2) { + if (iface->firmware_managed_reset) + return -EINVAL; + ret = gpiod_set_value_cansleep(iface->dchid->afe_reset, 1); + if (ret < 0) + return ret; + fsleep(50 * USEC_PER_MSEC); + } + ret = dchid_send_firmware(iface, fw, size); + if (ret < 0) { + dev_err(iface->dchid->dev, "Failed to send %s firmware\n", iface->name); + return ret; + } + + /* After loading firmware, multi-touch needs a reset */ + dev_dbg(iface->dchid->dev, "Resetting %s\n", iface->name); + if (iface->power_method == DCHID_POWER_METHOD_2) { + /* The retained session is already off while reset is asserted. */ + if (!iface->dchid->inherited_mtp) { + ret = dchid_pm2_set_power(iface, DCHID_POWER_STATE_OFF); + if (ret < 0) + return ret; + } + ret = dchid_pm2_set_power(iface, DCHID_POWER_STATE_ON); + if (ret < 0) + return ret; + } else { + ret = dchid_reset_interface(iface, DCHID_POWER_STATE_OFF); + /* After a failed AFE power-off the trackpad stays off: no retry. */ + if (ret >= 0 || !iface->dchid->afe_smc || iface->index != 1) + dchid_reset_interface(iface, DCHID_POWER_STATE_ON); + } + return 0; +} + static int dchid_start_interface(struct dchid_iface *iface) { void *fw; @@ -775,45 +825,9 @@ static int dchid_start_interface(struct dchid_iface *iface) /* Only multi-touch has firmware */ if (fw && size) { - - /* Send firmware to the device */ - dev_dbg(iface->dchid->dev, "Sending firmware for %s\n", iface->name); - /* Retained MTP must see AFE reset throughout firmware upload. */ - if (iface->dchid->inherited_mtp && - iface->power_method == DCHID_POWER_METHOD_2) { - if (iface->firmware_managed_reset) { - ret = -EINVAL; - goto err; - } - ret = gpiod_set_value_cansleep(iface->dchid->afe_reset, 1); - if (ret < 0) - goto err; - fsleep(50 * USEC_PER_MSEC); - } - ret = dchid_send_firmware(iface, fw, size); - if (ret < 0) { - dev_err(iface->dchid->dev, "Failed to send %s firmware\n", iface->name); + ret = dchid_load_firmware(iface, fw, size); + if (ret < 0) goto err; - } - - /* After loading firmware, multi-touch needs a reset */ - dev_dbg(iface->dchid->dev, "Resetting %s\n", iface->name); - if (iface->power_method == DCHID_POWER_METHOD_2) { - /* The retained session is already off while reset is asserted. */ - if (!iface->dchid->inherited_mtp) { - ret = dchid_pm2_set_power(iface, DCHID_POWER_STATE_OFF); - if (ret < 0) - goto err; - } - ret = dchid_pm2_set_power(iface, DCHID_POWER_STATE_ON); - if (ret < 0) - goto err; - } else { - ret = dchid_reset_interface(iface, DCHID_POWER_STATE_OFF); - /* After a failed AFE power-off the trackpad stays off: no retry. */ - if (ret >= 0 || !iface->dchid->afe_smc || iface->index != 1) - dchid_reset_interface(iface, DCHID_POWER_STATE_ON); - } } return 0; @@ -1531,11 +1545,23 @@ static int dchid_drain_ring(struct dockchannel_hid *dchid) dchid_ring_copy(dchid, dchid->pkt_buf, consumer, sizeof(struct dchid_hdr)); size = dchid_frame_size((void *)dchid->pkt_buf); + if (size < 0 && dchid->ring_resync) + goto resync; if (size < 0) return size; if (available < size) return 0; dchid_ring_copy(dchid, dchid->pkt_buf, consumer, size); + if (dchid->ring_resync) { + u32 checksum; + + if (!dchid_verify_checksum((void *)dchid->pkt_buf, + dchid->pkt_buf + sizeof(struct dchid_hdr), + &checksum)) + goto resync; + /* The first good frame shows the ring is in step. */ + dchid->ring_resync = false; + } dchid_dispatch(dchid, (void *)dchid->pkt_buf, dchid->pkt_buf + sizeof(struct dchid_hdr), false); consumer += size; @@ -1548,6 +1574,21 @@ static int dchid_drain_ring(struct dockchannel_hid *dchid) cond_resched(); } return 0; + +resync: + /* + * Only right after a resume from disk took the ring over: frames the + * firmware wrote while the image was being restored are lost. Drop + * what is between the consumer and the producer once. + */ + dchid->ring_resync = false; + dev_warn(dchid->dev, "receive ring out of step after resume from disk: skipping %#x..%#x\n", + consumer, producer); + dchid->ring_read = producer; + /* As in the loop: frame reads complete before the producer may reuse them. */ + mb(); + WRITE_ONCE(indices[1], producer); + return 0; } static void dchid_handle_packet(void *cookie, size_t avail) @@ -1559,9 +1600,12 @@ static void dchid_handle_packet(void *cookie, size_t avail) if (READ_ONCE(dchid->stopping)) return; + /* Woken with less than a header in the FIFO: nothing to read yet. */ + if (avail < sizeof(hdr)) + goto done; if (dockchannel_recv(dchid->dc, &hdr, sizeof(hdr)) != sizeof(hdr)) { dev_err(dchid->dev, "Read failed (header)\n"); - return; + goto done; } if (hdr.hdr_len != sizeof(hdr)) { dev_err(dchid->dev, "Bad header length %d\n", hdr.hdr_len); @@ -1913,8 +1957,290 @@ static int dockchannel_hid_resume(struct device *dev) return ret; } -static DEFINE_SIMPLE_DEV_PM_OPS(dockchannel_hid_pm_ops, dockchannel_hid_suspend, - dockchannel_hid_resume); +/* + * Resume from disk with the MTP started by the boot kernel (for instance to + * type a disk passphrase in its initramfs). The boot kernel quiesces its + * devices before it jumps into the image; the MTP firmware is not stopped + * (it does not come back from RTKit QUIESCED), so the restored kernel takes + * over the running instance, as the RTKit helper does for the RTKit session: + * + * boot kernel (.freeze while apple_rtkit_hib_restoring()): + * the input interfaces (keyboard; multi-touch once its firmware runs) are + * powered off with the interface power-state commands, so that the + * firmware sends (and DMAs) as little as possible while + * the image is restored; then receive stops, the receive ring is drained + * and its position is left for the image with the ring's IOVA. + * + * restored kernel (.restore), only if the helper took the RTKit session + * over and the ring sits at the same IOVA in both kernels: + * the ring indices are set to the position the boot kernel stopped at + * (or, if the boot kernel never registered the ring, the ring is + * registered again from its start), receive restarts, and the interfaces + * the boot kernel powered off are powered on again. A multi-touch + * interface the image had started but the boot kernel had not gets its + * firmware as on a first open. + * + * Otherwise .restore keeps the plain behaviour: a co-processor the boot + * loader left halted stops the transport until userspace reloads the + * drivers, one that runs but cannot be adopted is left as it is. + * Only the ring transport (T6041 / J616s) is handled. + */ +struct dchid_hib { + u32 version; + u32 ring_size; + u64 ring_dma; + u32 ring_pos; + u32 off_mask; + u32 started_mask; + u32 flags; +}; + +#define DCHID_HIB_VERSION 2 +/* The boot kernel gave its firmware the receive ring. */ +#define DCHID_HIB_RING_REGISTERED BIT(0) + +static void dchid_hib_tag(struct dockchannel_hid *dchid, char *tag, size_t len) +{ + snprintf(tag, len, "dchid:%s", dev_name(dchid->dev)); +} + +static bool dchid_hib_input(struct dchid_iface *iface) +{ + if (!strcmp(iface->name, "keyboard")) + return true; + /* Without its firmware the multi-touch interface was never powered. */ + return !strcmp(iface->name, "multi-touch") && iface->starting; +} + +/* The power sequences of dchid_load_firmware(), one direction at a time. */ +static int dchid_hib_set_power(struct dchid_iface *iface, u8 state) +{ + if (iface->power_method == DCHID_POWER_METHOD_2) + return dchid_pm2_set_power(iface, state); + return dchid_reset_interface(iface, state); +} + +static void dchid_hib_quiesce(struct dockchannel_hid *dchid) +{ + struct dchid_hib rec = { + .version = DCHID_HIB_VERSION, + .ring_size = DCHID_RING_SIZE, + .ring_dma = dchid->ring_dma, + }; + u32 *indices = dchid->ring; + char tag[32]; + int i, ret; + + for (i = 0; i < MAX_INTERFACES; i++) { + struct dchid_iface *iface = dchid->ifaces[i]; + + if (!iface || !iface->hid) + continue; + if (iface->starting) + rec.started_mask |= BIT(i); + if (!dchid_hib_input(iface)) + continue; + ret = dchid_hib_set_power(iface, DCHID_POWER_STATE_OFF); + if (ret < 0) { + dev_warn(dchid->dev, "resume from disk: %s not powered off: %d\n", + iface->name, ret); + continue; + } + rec.off_mask |= BIT(i); + } + + if (READ_ONCE(dchid->ring_registered) && !dchid->ring_error) + rec.flags |= DCHID_HIB_RING_REGISTERED; + + /* The power-off acknowledgments came through the ring: stop now. */ + dockchannel_await(dchid->dc, NULL, NULL, 0); + ret = dchid_drain_ring(dchid); + if (ret < 0) + dev_warn(dchid->dev, "resume from disk: final ring drain failed: %d\n", ret); + rec.ring_pos = dchid->ring_read; + if (READ_ONCE(indices[0]) != rec.ring_pos) + dev_warn(dchid->dev, "resume from disk: ring not empty at hand-off (%#x, producer %#x)\n", + rec.ring_pos, READ_ONCE(indices[0])); + + dchid->hib_quiesced = true; + dchid->hib_off_mask = rec.off_mask; + dchid_hib_tag(dchid, tag, sizeof(tag)); + ret = apple_rtkit_hib_handoff_put(tag, &rec, sizeof(rec)); + if (ret) + dev_warn(dchid->dev, "resume from disk: no hand-off record for the image: %d\n", + ret); + else + dev_info(dchid->dev, "resume from disk: input left for the image (interfaces off %#x, ring at %#x)\n", + rec.off_mask, rec.ring_pos); +} + +static int dockchannel_hid_freeze(struct device *dev) +{ + struct dockchannel_hid *dchid = dev_get_drvdata(dev); + int ret; + + ret = dockchannel_hid_suspend(dev); + if (ret) + return ret; + if (dchid->use_ring && apple_rtkit_hib_restoring()) + dchid_hib_quiesce(dchid); + return 0; +} + +static void dchid_hib_power_on(struct dockchannel_hid *dchid, u32 mask) +{ + int i, ret; + + for (i = 0; i < MAX_INTERFACES; i++) { + struct dchid_iface *iface = dchid->ifaces[i]; + + if (!(mask & BIT(i)) || !iface) + continue; + ret = dchid_hib_set_power(iface, DCHID_POWER_STATE_ON); + if (ret < 0) + dev_err(dchid->dev, "resume from disk: %s not powered on: %d\n", + iface->name, ret); + } +} + +static int dockchannel_hid_thaw(struct device *dev) +{ + struct dockchannel_hid *dchid = dev_get_drvdata(dev); + char tag[32]; + + /* The jump into the image failed: this kernel keeps the input. */ + if (dchid->hib_quiesced) { + dchid->hib_quiesced = false; + dchid_hib_tag(dchid, tag, sizeof(tag)); + apple_rtkit_hib_handoff_drop(tag); + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, + sizeof(struct dchid_hdr)); + dchid_hib_power_on(dchid, dchid->hib_off_mask); + } + return dockchannel_hid_resume(dev); +} + +static void dchid_hib_take_over(struct dockchannel_hid *dchid, const struct dchid_hib *rec) +{ + const u32 capacity = DCHID_RING_SIZE - DCHID_RING_HEADER_SIZE; + u32 *indices = dchid->ring; + bool reregister; + int i, ret; + + if (rec->version != DCHID_HIB_VERSION || rec->ring_size != DCHID_RING_SIZE || + rec->ring_dma != dchid->ring_dma || rec->ring_pos >= capacity || + !IS_ALIGNED(rec->ring_pos, 4)) { + dev_err(dchid->dev, "resume from disk: boot kernel ring %#llx+%#x@%#x does not match the image's %pad: input not taken over\n", + rec->ring_dma, rec->ring_size, rec->ring_pos, &dchid->ring_dma); + return; + } + + /* + * A boot kernel that started no interface (nothing opened the trackpad) + * never gave its firmware the receive ring, which this kernel had + * registered with its own: register it again, from the start, as a + * first open does. Otherwise the firmware goes on at the position the + * boot kernel drained to; the ring memory and indices it wrote were in + * the boot kernel's RAM. + */ + reregister = !(rec->flags & DCHID_HIB_RING_REGISTERED) && + READ_ONCE(dchid->ring_registered); + dockchannel_await(dchid->dc, NULL, NULL, 0); + mutex_lock(&dchid->ring_mutex); + if (reregister) { + dchid->ring_read = 0; + WRITE_ONCE(indices[0], 0); + WRITE_ONCE(indices[1], 0); + dchid->ring_resync = false; + dchid->ring_error = 0; + WRITE_ONCE(dchid->ring_registered, false); + } else { + dchid->ring_read = rec->ring_pos; + WRITE_ONCE(indices[0], rec->ring_pos); + WRITE_ONCE(indices[1], rec->ring_pos); + dchid->ring_resync = true; + } + mutex_unlock(&dchid->ring_mutex); + /* Indices visible to the firmware before it is told anything. */ + wmb(); + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, + sizeof(struct dchid_hdr)); + + if (reregister) { + ret = dchid_register_ring(dchid); + if (ret < 0) + dev_err(dchid->dev, "resume from disk: receive ring not registered with the boot kernel's firmware: %d\n", + ret); + else + dev_info(dchid->dev, "resume from disk: receive ring registered with the boot kernel's firmware\n"); + } + + dchid_hib_power_on(dchid, rec->off_mask); + + /* Started here but not in the boot kernel: firmware as on a first open. */ + for (i = 0; i < MAX_INTERFACES; i++) { + struct dchid_iface *iface = dchid->ifaces[i]; + void *fw; + size_t size; + + if (!iface || !iface->starting || (rec->started_mask & BIT(i))) + continue; + ret = dchid_get_firmware(iface, &fw, &size); + if (!ret && fw && size) + ret = dchid_load_firmware(iface, fw, size); + if (ret < 0) + dev_err(dchid->dev, "resume from disk: %s not restarted: %d\n", + iface->name, ret); + } + + dev_info(dchid->dev, "resume from disk: took over the input the boot kernel started (interfaces on %#x, ring at %#x)\n", + rec->off_mask, rec->ring_pos); +} + +static int dockchannel_hid_restore(struct device *dev) +{ + struct dockchannel_hid *dchid = dev_get_drvdata(dev); + struct dchid_hib rec; + char tag[32]; + int ret; + + dchid->hib_quiesced = false; + dchid_hib_tag(dchid, tag, sizeof(tag)); + ret = apple_rtkit_hib_handoff_take(tag, &rec, sizeof(rec)); + if (ret == sizeof(rec)) { + if (!dchid->use_ring) + dev_warn(dev, "resume from disk: hand-off without a receive ring: input not taken over\n"); + else if (!apple_rtkit_helper_readopted(dchid->helper_link->supplier)) + dev_warn(dev, "resume from disk: the RTKit helper did not take the session over: input not taken over\n"); + else + dchid_hib_take_over(dchid, &rec); + } else if (ret != -ENOENT) { + dev_warn(dev, "resume from disk: invalid hand-off record: %d\n", ret); + } + + /* + * The boot loader halted the co-processor and the boot kernel did not + * start it: the firmware this kernel talked to is gone, and the FIFO + * never answers again. Stop the transport, so that receive does not + * wait for frames and the HID core's requests fail at once, until the + * drivers are probed again (which starts the co-processor). + */ + if (apple_rtkit_helper_halted_underneath(dchid->helper_link->supplier)) { + WRITE_ONCE(dchid->stopping, true); + dockchannel_await(dchid->dc, NULL, NULL, 0); + dev_info(dev, "resume from disk: co-processor halted underneath: input stopped until the drivers are reloaded\n"); + } + return dockchannel_hid_resume(dev); +} + +static const struct dev_pm_ops dockchannel_hid_pm_ops = { + .suspend = pm_sleep_ptr(dockchannel_hid_suspend), + .resume = pm_sleep_ptr(dockchannel_hid_resume), + .freeze = pm_sleep_ptr(dockchannel_hid_freeze), + .thaw = pm_sleep_ptr(dockchannel_hid_thaw), + .poweroff = pm_sleep_ptr(dockchannel_hid_suspend), + .restore = pm_sleep_ptr(dockchannel_hid_restore), +}; static const struct of_device_id dockchannel_hid_of_match[] = { { .compatible = "apple,dockchannel-hid" }, diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs index e64f7ff26a6e0..62eac075fa5ca 100644 --- a/drivers/iio/common/aop_sensors/aop_als.rs +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -4,15 +4,25 @@ //! //! Copyright (C) The Asahi Linux Contributors +use core::{ + ffi::{c_int, c_void}, + ptr, +}; + use kernel::{ bindings, c_str, - device::Core, + device::{self, Core}, firmware::Firmware, iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor, MICRO}, module_platform_driver, of, platform, prelude::*, soc::apple::aop::{EPICService, FakehidRegistration, AOP}, - sync::Arc, + sync::{ + aref::ARef, + atomic::{Atomic, Relaxed}, + Arc, + }, + types::Opaque, }; const EPIC_SUBTYPE_GET_AOP_PROPERTY: u16 = 0xa; @@ -146,6 +156,149 @@ fn set_als_property(aop: &dyn AOP, svc: &EPICService, tag: u32, data: &[u8]) -> aop.epic_call(svc, EPIC_SUBTYPE_SET_ALS_PROPERTY, &buf) } +/// Stops the reports around system sleep and starts them again after. +/// +/// The AOP mailbox interrupt stays enabled in suspend-to-idle, so every +/// report ends the CPUs' idle period: at the 200 ms interval the sensor +/// woke the system five times a second. A zero interval stops the reports: +/// it is written when the system sleeps and the previous interval when it +/// wakes. +/// +/// Only suspend is handled. A hibernation powers the system off, and after +/// a resume from disk the AOP core may have restarted the co-processor and +/// probed this driver again, which then starts the reports itself. +#[repr(C)] +struct PmHook { + /// First, so that the callback finds the hook from it. + nb: Opaque, + dev: ARef, + aop: Arc, + service: EPICService, + registered: Atomic, +} + +/// The PM notifier callback. +/// +/// # Safety +/// +/// `nb` is the notifier block of a registered [`PmHook`]. +unsafe extern "C" fn pm_event( + nb: *mut bindings::notifier_block, + action: usize, + _data: *mut c_void, +) -> c_int { + // SAFETY: `nb` is the first field of a registered `PmHook`. + let hook = unsafe { &*nb.cast::() }; + let interval = match action as u32 { + bindings::PM_SUSPEND_PREPARE => 0, + bindings::PM_POST_SUSPEND => ALS_INTERVAL_US, + _ => return bindings::NOTIFY_DONE as c_int, + }; + // As at probe, only the transport can fail; the sensor's status word is + // not an error (see the property comments above). + match set_als_property( + hook.aop.as_ref(), + &hook.service, + ALS_PROP_INTERVAL, + &interval.to_le_bytes(), + ) { + Ok(status) => dev_dbg!( + hook.dev, + "interval = {} for sleep: status {:#x}\n", + interval, + status + ), + Err(e) => dev_warn!(hook.dev, "reports not switched for sleep: {:?}\n", e), + } + bindings::NOTIFY_OK as c_int +} + +/// # Safety +/// +/// `nb` stays valid and pinned until [`pm_unregister`] is called on it. +#[cfg(CONFIG_PM_SLEEP)] +unsafe fn pm_register(nb: *mut bindings::notifier_block) -> c_int { + // SAFETY: By the caller's contract. + unsafe { bindings::register_pm_notifier(nb) } +} + +/// # Safety +/// +/// `nb` was registered by [`pm_register`]. +#[cfg(CONFIG_PM_SLEEP)] +unsafe fn pm_unregister(nb: *mut bindings::notifier_block) { + // SAFETY: By the caller's contract. + unsafe { bindings::unregister_pm_notifier(nb) }; +} + +/// Without system sleep there is nothing to switch. +/// +/// # Safety +/// +/// None; the signature matches the one with system sleep. +#[cfg(not(CONFIG_PM_SLEEP))] +unsafe fn pm_register(_nb: *mut bindings::notifier_block) -> c_int { + 0 +} + +/// # Safety +/// +/// None; the signature matches the one with system sleep. +#[cfg(not(CONFIG_PM_SLEEP))] +unsafe fn pm_unregister(_nb: *mut bindings::notifier_block) {} + +// SAFETY: The notifier block is only touched by the PM core, which +// serializes registration, removal and the callbacks itself. +unsafe impl Send for PmHook {} +// SAFETY: See above; the other fields are `Sync`. +unsafe impl Sync for PmHook {} + +impl PmHook { + fn new( + dev: &device::Device, + aop: Arc, + service: EPICService, + ) -> Result>> { + let hook = KBox::pin( + Self { + nb: Opaque::new(bindings::notifier_block { + notifier_call: Some(pm_event), + next: ptr::null_mut(), + priority: 0, + }), + dev: dev.into(), + aop, + service, + registered: Atomic::new(false), + }, + GFP_KERNEL, + )?; + // SAFETY: The block is pinned inside the hook, which unregisters it + // before it goes. + let ret = unsafe { pm_register(hook.nb.get()) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + hook.registered.store(true, Relaxed); + Ok(hook) + } + + /// Returns once no callback is running any more; repeated calls do + /// nothing. + fn unregister(&self) { + if self.registered.xchg(false, Relaxed) { + // SAFETY: `nb` was registered by `new()`. + unsafe { pm_unregister(self.nb.get()) }; + } + } +} + +impl Drop for PmHook { + fn drop(&mut self) { + self.unregister(); + } +} + /// Converts the bits of an IEEE 754 single to microlux, rounding down. /// Negative values clamp to zero; NaN and the infinities carry no reading. /// Values beyond `MAX_MICRO` saturate when they are stored. @@ -206,7 +359,9 @@ enum AlsIio { } struct IIOAopAlsDriver { - /// Declared first: the subscription ends before the IIO device goes. + /// Fields are declared in teardown order: sleep handling first, then the + /// subscription, which ends before the IIO device goes. + pm: Pin>, listener: FakehidRegistration, _iio: AlsIio, } @@ -285,13 +440,16 @@ impl platform::Driver for IIOAopAlsDriver { }, ) }; + let pm = PmHook::new(dev, adata, service)?; Ok(IIOAopAlsDriver { + pm, listener, _iio: iio, }) } fn unbind(_dev: &platform::Device>, this: Pin<&Self>) { + this.pm.unregister(); this.listener.unregister(); } } diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index a8b91c7e63316..b046fc8aa8388 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -414,6 +414,8 @@ struct apple_dart { u32 restore_on_power_up : 1; /* Powered down by its DCP across a real sleep: see apple_dart_host_power_on(). */ u32 host_off_slept : 1; + /* A PCIe-C DART that loses its probe tunables with its state (T6040, T6041). */ + u32 reapply_tunables : 1; bool commands_gated; /* Resume restored the translations, but the last steps of its reset failed. */ bool reset_incomplete; @@ -534,6 +536,14 @@ struct apple_dart { * The first error of each stream is logged. */ int locked_retire_err[DART_MAX_STREAMS]; + /* Physical address of each mapped locked root, as read from its TTBR. */ + phys_addr_t locked_phys[DART_MAX_STREAMS][DART_MAX_TTBR]; + /* + * Page-table configuration last published into each locked stream's + * roots, for the re-import after a resume from disk (the boot loader + * installs fresh firmware roots). Cleared when the stream's roots retire. + */ + struct io_pgtable_cfg *locked_cfg[DART_MAX_STREAMS]; }; #define APPLE_DART_WRITER_COUNTED 1 @@ -1463,6 +1473,7 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) dart->locked_fw[sid][idx] = fw; if (!idx) dart->locked_fw_tree[sid] = tree; + dart->locked_phys[sid][idx] = phys; spin_unlock_irqrestore(&dart->lock, flags); } @@ -1513,6 +1524,8 @@ apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx tree = dart->locked_fw_tree[sid]; dart->locked_fw_tree[sid] = NULL; } + dart->locked_phys[sid][idx] = 0; + dart->locked_cfg[sid] = NULL; spin_unlock_irqrestore(&dart->lock, flags); /* Once per stream: the error is never cleared. */ if (first) @@ -1612,6 +1625,8 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, if (ret) goto out; } + if (dart->locked_ttbr[sid][0]) + dart->locked_cfg[sid] = cfg; } out: spin_unlock_irqrestore(&dart->lock, flags); @@ -5346,6 +5361,8 @@ static int apple_dart_probe(struct platform_device *pdev) dev_dbg(dev, "applying %zu PCIe-C DART tunables\n", dart->tunables->sz); apple_dart_apply_tunables(dart, dart->tunables); + dart->reapply_tunables = of_device_is_compatible(dev->of_node, + "apple,t6041-dart"); } if (tunneled) { @@ -5365,8 +5382,13 @@ static int apple_dart_probe(struct platform_device *pdev) dart->oas = DART_PCIEC_ADDR_WIDTH; dart->num_streams = DART_PCIEC_STREAMS; dart->four_level = true; - /* T8132 PCIe-C DARTs keep per-stream exception records. */ - if (of_device_is_compatible(dev->of_node, "apple,t8132-dart")) + /* + * T8132 PCIe-C DARTs keep per-stream exception records. + * T6040/T6041 carry the same PCIe-C block, and their DARTs + * report version 0x0202 (PARAMS3) wherever they can be read. + */ + if (of_device_is_compatible(dev->of_node, "apple,t8132-dart") || + of_device_is_compatible(dev->of_node, "apple,t6041-dart")) dart->version = 0x0202; } else { dart->pgsize = 1 << DART_PCIEC_PAGE_SHIFT; @@ -6486,6 +6508,13 @@ static __maybe_unused int apple_dart_resume(struct device *dev) return ret; } + /* + * A T6040/T6041 PCIe-C DART that lost its state lost its tunables with it: + * apply them again before the reset, in the order of probe. + */ + if (dart->reapply_tunables) + apple_dart_apply_tunables(dart, dart->tunables); + ret = apple_dart_hw_reset(dart); /* @@ -6676,8 +6705,135 @@ static __maybe_unused int apple_dart_sys_resume_noirq(struct device *dev) return pm_suspend_via_firmware() ? apple_dart_resume_noirq(dev) : 0; } +/* + * Resume from disk is a cold boot: the boot loader locked every locked DART + * again with fresh firmware-owned roots, while the restored kernel still + * describes the roots of the image (their mapping, their firmware snapshot + * and the slots it had published). Every publication then fails (-EBUSY), + * and with it every DMA mapping through the DART. Re-import the live roots + * the way attach maps them, then publish the stream's page tables into them + * again, as attach does. Firmware-owned memory is never written: the old root's + * published slots are forgotten, not retired. + */ +static int apple_dart_reimport_locked(struct apple_dart *dart) +{ + DECLARE_BITMAP(redo, DART_MAX_STREAMS) = {}; + unsigned int sid, idx; + int ret = 0, done = 0; + + /* + * Two passes, as attach does: every stream's live roots are re-read and + * snapshotted before any of them is republished, so that a root shared + * by several streams is snapshotted before Linux slots enter it. + */ + for (sid = 0; sid < dart->num_streams; sid++) { + bool any = false; + + for (idx = 0; idx < dart->hw->ttbr_count; idx++) { + u64 *live = dart->locked_ttbr[sid][idx]; + u64 *owned = dart->locked_owned[sid][idx]; + struct apple_dart_fw_root *fw, *old_fw; + unsigned long flags; + phys_addr_t phys; + u64 *l1 = live; + u32 ttbr; + + if (!live || !owned) + continue; + ttbr = apple_dart_readl(dart, DART_TTBR(dart, sid, idx)); + if (!(ttbr & dart->hw->ttbr_valid)) { + dev_err(dart->dev, "hibernation restore: stream %u ttbr[%u] not valid\n", + sid, idx); + ret = ret ?: -EIO; + continue; + } + ttbr &= ~dart->hw->ttbr_valid; + if (dart->hw->ttbr_addr_field_shift) + ttbr >>= dart->hw->ttbr_addr_field_shift; + phys = ((phys_addr_t)ttbr) << dart->hw->ttbr_shift; + if (dart->fw_handoff && !apple_dart_fw_table_ram(dart, phys)) { + ret = ret ?: -EINVAL; + continue; + } + if (phys != dart->locked_phys[sid][idx]) { + l1 = devm_memremap(dart->dev, phys, dart->pgsize, MEMREMAP_WB); + if (IS_ERR(l1)) { + ret = ret ?: PTR_ERR(l1); + continue; + } + } + fw = apple_dart_snapshot_fw_root(dart, sid, l1); + if (IS_ERR(fw)) { + if (l1 != live) + devm_memunmap(dart->dev, l1); + ret = ret ?: PTR_ERR(fw); + continue; + } + spin_lock_irqsave(&dart->lock, flags); + old_fw = dart->locked_fw[sid][idx]; + memset(owned, 0, dart->pgsize); + dart->locked_ttbr[sid][idx] = l1; + dart->locked_phys[sid][idx] = phys; + dart->locked_fw[sid][idx] = fw; + spin_unlock_irqrestore(&dart->lock, flags); + apple_dart_free_fw_root(dart, old_fw); + if (l1 != live) + devm_memunmap(dart->dev, live); + any = true; + } + if (any) + __set_bit(sid, redo); + } + + for_each_set_bit(sid, redo, dart->num_streams) { + struct apple_dart_stream_map stream = { .dart = dart }; + struct io_pgtable_cfg *cfg = READ_ONCE(dart->locked_cfg[sid]); + int err; + + if (!cfg) + continue; + __set_bit(sid, stream.sidmap); + err = apple_dart_hw_sync_locked(cfg, &stream, false); + if (!err) + err = dart->hw->invalidate_tlb(&stream); + if (err) { + dev_err(dart->dev, "hibernation restore: stream %u not republished: %d\n", + sid, err); + ret = ret ?: err; + continue; + } + done++; + } + dev_dbg(dart->dev, "hibernation restore: %d locked streams re-imported from the boot loader's roots, first error %d\n", + done, ret); + return ret; +} + +static int apple_dart_restore(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_force_resume(dev); + if (ret || !dart->locked) + return ret; + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return ret; + /* Logged per stream; a failed stream must not fail the whole restore. */ + apple_dart_reimport_locked(dart); + pm_runtime_put(dev); + return 0; +} + +/* A resume from disk also re-imports the roots the boot loader locked again. */ static const struct dev_pm_ops apple_dart_pm_ops = { - SYSTEM_SLEEP_PM_OPS(apple_dart_system_suspend, apple_dart_system_resume) + .suspend = pm_sleep_ptr(apple_dart_system_suspend), + .resume = pm_sleep_ptr(apple_dart_system_resume), + .freeze = pm_sleep_ptr(apple_dart_system_suspend), + .thaw = pm_sleep_ptr(apple_dart_system_resume), + .poweroff = pm_sleep_ptr(apple_dart_system_suspend), + .restore = pm_sleep_ptr(apple_dart_restore), NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_dart_sys_suspend_noirq, apple_dart_sys_resume_noirq) RUNTIME_PM_OPS(apple_dart_suspend, apple_dart_resume, NULL) }; diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 57a5f868a3606..a94029884eddb 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -349,6 +349,18 @@ static const struct aic_info aic3_t8132_info __initconst = { .fw_suspend = true, }; +/* + * The same AICv3 as T8140's across a firmware-driven suspend: the CONFIG, + * TIMEOUT and CLUSTER_PRIO words and the IRQ_CFG array are lost with its power. + */ +static const struct aic_info aic3_t6041_info __initconst = { + .version = 3, + .irq_cfg = AIC3_IRQ_CFG, + .fast_ipi = true, + .local_fast_ipi = true, + .fw_suspend = true, +}; + static const struct aic_info aic3_t8142_info __initconst = { .version = 3, .irq_cfg = AIC3_IRQ_CFG, @@ -366,6 +378,10 @@ static const struct of_device_id aic_info_match[] = { .compatible = "apple,t8140-aic3", .data = &aic3_t8140_info, }, + { + .compatible = "apple,t6041-aic3", + .data = &aic3_t6041_info, + }, { .compatible = "apple,t8142-aic3", .data = &aic3_t8142_info, @@ -1165,7 +1181,8 @@ static int aic_syscore_suspend(void *data) struct aic_irq_chip *ic = aic_irqc; u32 die, i, off = 0; - if (!pm_suspend_via_firmware()) + /* Registered for the resume from disk alone where the SoC has no such suspend. */ + if (!ic->info.fw_suspend || !pm_suspend_via_firmware()) return 0; aic_sleep.config = aic_ic_read(ic, AIC2_CONFIG); @@ -1205,13 +1222,45 @@ static void aic_resume_boot_cpu(void) } } +/* + * After a resume from disk the controller holds what the boot kernel left: its + * init masked every IRQ and it unmasked only the ones it used itself. The IRQ + * core disables lazily and unmasks only an IRQ it recorded as masked, so on + * its own it never unmasks the restored kernel's other IRQs again. Make every + * hardware mask match the restored descriptor: unmasked unless the core + * recorded it masked (a lazily disabled IRQ was unmasked in hardware when the + * image was taken), and masked when this kernel has no mapping for it. + */ +static void aic_resync_masks(struct aic_irq_chip *ic) +{ + unsigned int die, i; + + for (die = 0; die < ic->nr_die; die++) { + for (i = 0; i < ic->nr_irq; i++) { + irq_hw_number_t hwirq = AIC_IRQ_HWIRQ(die, i); + unsigned int virq = irq_find_mapping(ic->hw_domain, hwirq); + struct irq_data *d = virq ? irq_get_irq_data(virq) : NULL; + u32 off = die * ic->info.die_stride; + + if (d && !irqd_irq_masked(d)) + aic_ic_write(ic, ic->info.mask_clr + off + MASK_REG(i), MASK_BIT(i)); + else + aic_ic_write(ic, ic->info.mask_set + off + MASK_REG(i), MASK_BIT(i)); + } + } +} + static void aic_syscore_resume(void *data) { struct aic_irq_chip *ic = aic_irqc; u32 die, i, off = 0; - if (!aic_sleep.saved) + if (!aic_sleep.saved) { + /* No firmware-driven suspend: only a resume from disk has lost the masks. */ + if (pm_hibernate_is_restored()) + aic_resync_masks(ic); return; + } aic_sleep.saved = false; /* Otherwise the suspend was refused, and the controller stayed powered. */ @@ -1254,14 +1303,21 @@ static struct syscore aic_syscore = { .ops = &aic_syscore_ops, }; +/* + * The save across a firmware-driven suspend is for the SoCs that have one. The + * mask resync after a resume from disk is for every SoC: whatever the AIC, the + * kernel that loaded the image left its own masks in it. + */ static void __init aic_sleep_init(struct aic_irq_chip *irqc) { - if (!irqc->info.fw_suspend) + if (!irqc->info.fw_suspend && !IS_ENABLED(CONFIG_HIBERNATION)) return; - aic_sleep.irq_cfg = kcalloc(irqc->nr_die * irqc->nr_irq, sizeof(u32), GFP_KERNEL); - if (WARN_ON(!aic_sleep.irq_cfg)) - return; + if (irqc->info.fw_suspend) { + aic_sleep.irq_cfg = kcalloc(irqc->nr_die * irqc->nr_irq, sizeof(u32), GFP_KERNEL); + if (WARN_ON(!aic_sleep.irq_cfg)) + return; + } register_syscore(&aic_syscore); } diff --git a/drivers/mfd/macsmc-test.c b/drivers/mfd/macsmc-test.c index a2dc3531ac36f..29920e5c482c8 100644 --- a/drivers/mfd/macsmc-test.c +++ b/drivers/mfd/macsmc-test.c @@ -10,9 +10,10 @@ struct macsmc_test_context { unsigned int calls; bool require_sram_lock; /* What the sleep tests saw, in order: keys read, values written, notices. */ - u32 seen[8]; + u32 seen[12]; unsigned int seen_count; int ap_power_result; + int atomic_result; u64 record; }; @@ -26,9 +27,11 @@ static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, KUNIT_EXPECT_TRUE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); ctx->calls++; + /* A written value as wide as it was written: WKTP's is one byte. */ if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) - ctx->seen[ctx->seen_count++] = cmd == SMC_MSG_WRITE_KEY ? - get_unaligned((u32 *)ctx->shmem) : arg; + ctx->seen[ctx->seen_count++] = cmd != SMC_MSG_WRITE_KEY ? arg : + get_unaligned((u32 *)ctx->shmem) & + GENMASK(8 * min_t(u64, size, sizeof(u32)) - 1, 0); if (ret_data && ctx->result >= 0) *ret_data = ctx->data; return ctx->result; @@ -222,12 +225,15 @@ static int macsmc_test_ap_power(struct apple_smc *smc, bool on) return ctx->ap_power_result; } -static void macsmc_sleep_order_test(struct kunit *test) +/* A sleep and its wake, for an SMC that takes WKTP and for one that does not. */ +static void macsmc_sleep_order(struct kunit *test, bool wktp) { struct macsmc_test_context *ctx = test->priv; + unsigned int n = 0; kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); ctx->result = 8; + ctx->smc.wktp = wktp; apple_smc_sleep_warn(&ctx->smc); KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), 0); @@ -236,15 +242,34 @@ static void macsmc_sleep_order_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); - /* As in the trace, values included. */ - KUNIT_ASSERT_EQ(test, ctx->seen_count, 7U); - KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); - KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(slpa)); - KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_IDLE); - KUNIT_EXPECT_EQ(test, ctx->seen[3], (u32)MACSMC_TEST_AP_ON); - KUNIT_EXPECT_EQ(test, ctx->seen[4], SMC_KEY(MBSE)); - KUNIT_EXPECT_EQ(test, ctx->seen[5], SMC_KEY(MBSW)); - KUNIT_EXPECT_EQ(test, ctx->seen[6], SMC_KEY(waka)); + /* + * Values included: WKTP's 2 comes before 'slpa', its 0 ends the wake. + * The lid is read at every wake, for the display. + */ + KUNIT_ASSERT_EQ(test, ctx->seen_count, wktp ? 10U : 8U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpw)); + if (wktp) + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 2U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(waka)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MSLD)); + if (wktp) + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 0U); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); +} + +static void macsmc_sleep_order_test(struct kunit *test) +{ + macsmc_sleep_order(test, false); +} + +static void macsmc_sleep_order_wktp_test(struct kunit *test) +{ + macsmc_sleep_order(test, true); } static void macsmc_test_outbox_poll(struct apple_smc *smc, const char *after, @@ -259,33 +284,274 @@ static void macsmc_sleep_warn_late_test(struct kunit *test) kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); kunit_activate_static_stub(test, apple_smc_outbox_poll, macsmc_test_outbox_poll); ctx->result = 8; + ctx->smc.wktp = true; - /* A sleep that cuts the debug link: slpw comes at the noirq suspend, before slpa. */ + /* Without last stops, slpw comes at the noirq suspend, right before the commit. */ ctx->smc.sleep_warn_late = true; KUNIT_EXPECT_EQ(test, apple_smc_sleep_enter(&ctx->smc), 0); KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warn_late); KUNIT_EXPECT_TRUE(test, ctx->smc.sleep_warned); - KUNIT_ASSERT_EQ(test, ctx->seen_count, 3U); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 4U); KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); - KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(slpa)); - KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[1], 2U); + KUNIT_EXPECT_EQ(test, ctx->seen[2], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[3], (u32)MACSMC_TEST_AP_IDLE); } -static void macsmc_sleep_refused_test(struct kunit *test) +static void macsmc_sleep_refused(struct kunit *test, bool wktp) { struct macsmc_test_context *ctx = test->priv; + unsigned int n = 0; kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); ctx->ap_power_result = -ETIME; ctx->result = 4; + ctx->smc.wktp = wktp; /* No answer: refuse, undo the notice and leave commands working. */ KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), -ETIME); KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); - KUNIT_ASSERT_EQ(test, ctx->seen_count, 3U); - KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpa)); - KUNIT_EXPECT_EQ(test, ctx->seen[1], (u32)MACSMC_TEST_AP_IDLE); - KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_ON); + KUNIT_ASSERT_EQ(test, ctx->seen_count, wktp ? 4U : 3U); + if (wktp) + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 2U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_ON); + + /* The key the commit wrote is taken back when the sleep ends, once. */ + KUNIT_EXPECT_TRUE(test, ctx->smc.wake_silent); + apple_smc_wake_full(&ctx->smc); + apple_smc_wake_full(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + KUNIT_ASSERT_EQ(test, ctx->seen_count, wktp ? 5U : 3U); + if (wktp) + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 0U); +} + +static void macsmc_sleep_refused_test(struct kunit *test) +{ + macsmc_sleep_refused(test, false); +} + +static void macsmc_sleep_refused_wktp_test(struct kunit *test) +{ + macsmc_sleep_refused(test, true); +} + +static bool macsmc_test_no_commit(void) +{ + return false; +} + +static void macsmc_sleep_dry_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_sleep_commits, macsmc_test_no_commit); + ctx->result = 8; + ctx->smc.wktp = true; + + /* + * A sleep that ends short of the power loss, as a dry run does: the SMC + * gets neither the warning nor anything of the commit. + */ + ctx->smc.sleep_warn_late = true; + KUNIT_EXPECT_EQ(test, apple_smc_sleep_noirq(&ctx->smc), 0); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + apple_smc_wake_noirq(&ctx->smc); + apple_smc_sleep_end(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warn_late); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 0U); +} + +static bool macsmc_test_commit(void) +{ + return true; +} + +static void macsmc_sleep_refused_late_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + unsigned int n = 0; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_outbox_poll, macsmc_test_outbox_poll); + kunit_activate_static_stub(test, apple_smc_sleep_commits, macsmc_test_commit); + ctx->result = 8; + ctx->smc.wktp = true; + + /* + * A real sleep, committed in the noirq suspend, that something refuses after the + * commit: no power loss, and the SMC gets what a wake sends, in a wake's order and + * once. + */ + apple_smc_sleep_warn(&ctx->smc); + KUNIT_EXPECT_EQ(test, apple_smc_sleep_noirq(&ctx->smc), 0); + KUNIT_EXPECT_TRUE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_TRUE(test, ctx->smc.wake_silent); + apple_smc_wake_noirq(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + apple_smc_sleep_end(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 10U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpw)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 2U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(waka)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MSLD)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 0U); +} + +/* The atomic writes of the sleep controller's last stop: no mutex, and only in atomic mode. */ +static int macsmc_test_write_atomic(struct apple_smc *smc, smc_key key, const void *buf, + size_t size) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + KUNIT_EXPECT_TRUE(kunit_get_current_test(), smc->atomic_mode); + KUNIT_EXPECT_FALSE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = size == sizeof(u8) ? *(const u8 *)buf : + get_unaligned((const u32 *)buf); + return ctx->atomic_result; +} + +static bool macsmc_test_at_stop(struct apple_smc *smc) +{ + return true; +} + +/* A sleep on a controller that runs last stops, up to where the noirq suspend has passed. */ +static void macsmc_last_stop_begin(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_outbox_poll, macsmc_test_outbox_poll); + kunit_activate_static_stub(test, apple_smc_sleep_commits, macsmc_test_commit); + kunit_activate_static_stub(test, apple_smc_commits_at_stop, macsmc_test_at_stop); + kunit_activate_static_stub(test, apple_smc_write_atomic, macsmc_test_write_atomic); + ctx->result = 8; + ctx->smc.wktp = true; + + /* The noirq suspend sends nothing and leaves the warning for the last stop. */ + ctx->smc.sleep_warn_late = true; + KUNIT_EXPECT_EQ(test, apple_smc_sleep_noirq(&ctx->smc), 0); + KUNIT_EXPECT_TRUE(test, ctx->smc.sleep_warn_late); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 0U); +} + +/* The wake after a last stop: a wake's commands in a wake's order, with the mutex again. */ +static void macsmc_last_stop_wake(struct kunit *test, unsigned int n) +{ + struct macsmc_test_context *ctx = test->priv; + + KUNIT_EXPECT_TRUE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_TRUE(test, ctx->smc.wake_silent); + KUNIT_EXPECT_FALSE(test, ctx->smc.atomic_mode); + KUNIT_EXPECT_FALSE(test, ctx->smc.atomic_pending); + apple_smc_wake_noirq(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + apple_smc_sleep_end(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + KUNIT_ASSERT_EQ(test, ctx->seen_count, n + 6); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(waka)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MSLD)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 0U); +} + +static void macsmc_last_stop_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + unsigned int n = 0; + + macsmc_last_stop_begin(test); + + /* The whole sequence comes from the last stop, in the noirq suspend's order, once. */ + KUNIT_EXPECT_EQ(test, apple_smc_last_stop(&ctx->smc), 0); + KUNIT_EXPECT_EQ(test, apple_smc_last_stop(&ctx->smc), 0); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warn_late); + KUNIT_EXPECT_TRUE(test, ctx->smc.sleep_warned); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 4U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpw)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 2U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_IDLE); + + macsmc_last_stop_wake(test, n); +} + +static void macsmc_last_stop_refused_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + macsmc_last_stop_begin(test); + + /* + * Refused after the noirq suspend and before the last stops, as by a wake event or + * the enclave: the SMC was sent nothing, and the resume sends it nothing. + */ + apple_smc_wake_noirq(&ctx->smc); + apple_smc_sleep_end(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warn_late); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 0U); + + /* A last stop with nothing left for it sends nothing either. */ + KUNIT_EXPECT_EQ(test, apple_smc_last_stop(&ctx->smc), 0); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 0U); +} + +static void macsmc_last_stop_failed_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + macsmc_last_stop_begin(test); + + /* + * An SMC that answers nothing: a last stop cannot refuse, but each further step would + * wait out its timeout again with interrupts off. Only the warning is tried, nothing else + * is sent, no idle notice is owed to the wake, and the first failure is returned. + */ + ctx->atomic_result = -ETIMEDOUT; + ctx->smc.atomic_pending = true; + KUNIT_EXPECT_EQ(test, apple_smc_last_stop(&ctx->smc), -ETIMEDOUT); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 1U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.atomic_mode); + KUNIT_EXPECT_FALSE(test, ctx->smc.atomic_pending); + KUNIT_EXPECT_TRUE(test, ctx->smc.sleep_warned); +} + +/* The same when the answer to an earlier command is still missing (-EBUSY). */ +static void macsmc_last_stop_late_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + macsmc_last_stop_begin(test); + + ctx->atomic_result = -EBUSY; + KUNIT_EXPECT_EQ(test, apple_smc_last_stop(&ctx->smc), -EBUSY); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 1U); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); } static void macsmc_notice_test(struct kunit *test) @@ -396,8 +662,16 @@ static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_sram_value_test), KUNIT_CASE(macsmc_key_info_test), KUNIT_CASE(macsmc_sleep_order_test), + KUNIT_CASE(macsmc_sleep_order_wktp_test), KUNIT_CASE(macsmc_sleep_warn_late_test), KUNIT_CASE(macsmc_sleep_refused_test), + KUNIT_CASE(macsmc_sleep_refused_wktp_test), + KUNIT_CASE(macsmc_sleep_dry_test), + KUNIT_CASE(macsmc_sleep_refused_late_test), + KUNIT_CASE(macsmc_last_stop_test), + KUNIT_CASE(macsmc_last_stop_refused_test), + KUNIT_CASE(macsmc_last_stop_failed_test), + KUNIT_CASE(macsmc_last_stop_late_test), KUNIT_CASE(macsmc_notice_test), KUNIT_CASE(macsmc_lid_wake_test), KUNIT_CASE(macsmc_wake_reason_test), diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 04e90a97cc313..c319d92ac8e1a 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +91,26 @@ static const struct mfd_cell apple_smc_t6050_described_devs[] = { MFD_CELL_OF("macsmc-rtc", NULL, NULL, 0, 0, "apple,smc-rtc"), }; +/* What this driver has to know of one SoC's SMC that the others do not share. */ +struct apple_smc_hw { + /* + * The SMC takes the WKTP key at the sleep request and once a wake is a + * full one: see apple_smc_sleep_commit() and apple_smc_wake_full(). + */ + bool wktp; + /* + * This SoC's sleep controller stops its coprocessors at the commit, behind + * every refusal, and the SMC's sleep commit goes with them: see + * apple_smc_last_stop(). + */ + bool commit_at_stop; +}; + +static const struct apple_smc_hw apple_smc_t6041_hw = { + .wktp = true, + .commit_at_stop = true, +}; + static int apple_smc_add_described_devices(struct apple_smc *smc, const struct mfd_cell *cells, size_t count) { @@ -356,11 +378,13 @@ EXPORT_SYMBOL(apple_smc_enter_atomic); int apple_smc_write_atomic(struct apple_smc *smc, smc_key key, const void *buf, size_t size) { + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_write_atomic, smc, key, buf, size); + guard(spinlock_irqsave)(&smc->lock); u8 result; int ret, poll_ret; - u64 msg; - ktime_t t0; + u64 msg, t0; s64 us; if (size > SMC_MAX_SIZE || size == 0) @@ -404,12 +428,13 @@ int apple_smc_write_atomic(struct apple_smc *smc, smc_key key, const void *buf, return ret; } - t0 = ktime_get(); + /* A sleep's last stop writes with timekeeping suspended, which ktime_get() warns of. */ + t0 = ktime_get_mono_fast_ns(); ret = read_poll_timeout_atomic(apple_rtkit_poll, poll_ret, poll_ret < 0 || !smc->atomic_pending, 100, SMC_ATOMIC_TIMEOUT_MS * USEC_PER_MSEC, false, smc->rtk); - us = ktime_us_delta(ktime_get(), t0); + us = div_u64(ktime_get_mono_fast_ns() - t0, NSEC_PER_USEC); if (us > 100 * USEC_PER_MSEC || ret < 0 || smc->atomic_pending) dev_warn(smc->dev, "atomic: key %p4ch reply after %lld us (ret %d)\n", &key, us, ret); @@ -572,11 +597,78 @@ static const struct apple_smc_wake_reason apple_smc_t8132_wake_reasons[] = { { 44, "USB2_wake" }, { 45, "aupo" }, { 48, "rtc" }, { } }; +/* + * The kernel that loads a hibernation image binds this driver like any boot + * and so holds a live RTKit session with the SMC when it jumps into the image. + * The restored kernel then asks the firmware to start again (IOP power INIT) + * and waits for a HELLO that a firmware already in a session never sends. So + * the loading kernel hands the SMC over quiesced, the state an unbind leaves + * it in (apple_smc_rtkit_shutdown()) and from which a wake starts a new + * session. + */ +static bool apple_smc_handed_over; + +/* + * The handshake with an SMC that was just woken, at probe and after a resume + * from disk alike: its endpoint, the initialization and the key count. The + * firmware is a new instance each time, and nothing says that its key table is + * the one of the instance before. + */ +static int apple_smc_handshake(struct apple_smc *smc) +{ + struct device *dev = smc->dev; + __be32 count = 0; + int ret; + + ret = apple_rtkit_start_ep(smc->rtk, SMC_ENDPOINT); + if (ret) { + dev_err(dev, "Failed to start SMC endpoint: %d\n", ret); + return ret; + } + + ret = apple_rtkit_send_message(smc->rtk, SMC_ENDPOINT, + FIELD_PREP(SMC_MSG, SMC_MSG_INITIALIZE), NULL, false); + if (ret) { + dev_err(dev, "Failed to send init message: %d\n", ret); + return ret; + } + + if (wait_for_completion_timeout(&smc->init_done, msecs_to_jiffies(SMC_TIMEOUT_MS)) == 0) { + dev_err(dev, "Timed out initializing SMC\n"); + return -ETIMEDOUT; + } + + if (smc->boot_stage != APPLE_SMC_INITIALIZED) { + dev_err(dev, "SMC failed to boot successfully, boot stage=%d\n", smc->boot_stage); + return -EIO; + } + + ret = apple_smc_read(smc, SMC_KEY(#KEY), &count, sizeof(count)); + if (ret >= 0 && ret != sizeof(count)) + ret = -EIO; + if (ret < 0) { + dev_err(dev, "Failed to get key count: %d\n", ret); + return ret; + } + smc->key_count = be32_to_cpu(count); + + return 0; +} + +static int apple_smc_last_stop(void *data); + +static void apple_smc_last_stop_unregister(void *data) +{ + struct apple_smc *smc = data; + + apple_sleep_coproc_unregister(smc->dev); +} + static int apple_smc_probe(struct platform_device *pdev) { + const struct apple_smc_hw *hw = device_get_match_data(&pdev->dev); struct device *dev = &pdev->dev; struct apple_smc *smc; - __be32 count = 0; bool t6050; int ret; @@ -585,6 +677,7 @@ static int apple_smc_probe(struct platform_device *pdev) return -ENOMEM; smc->wake_reasons = of_device_is_compatible(dev->of_node, "apple,t8132-smc") ? apple_smc_t8132_wake_reasons : apple_smc_t8140_wake_reasons; + smc->wktp = hw && hw->wktp; mutex_init(&smc->mutex); spin_lock_init(&smc->lock); @@ -606,39 +699,14 @@ static int apple_smc_probe(struct platform_device *pdev) if (ret) return ret; - ret = apple_rtkit_start_ep(smc->rtk, SMC_ENDPOINT); - if (ret) - return dev_err_probe(dev, ret, "Failed to start SMC endpoint"); - init_completion(&smc->init_done); init_completion(&smc->cmd_done); - - ret = apple_rtkit_send_message(smc->rtk, SMC_ENDPOINT, - FIELD_PREP(SMC_MSG, SMC_MSG_INITIALIZE), NULL, false); - if (ret) - return dev_err_probe(dev, ret, "Failed to send init message"); - - if (wait_for_completion_timeout(&smc->init_done, msecs_to_jiffies(SMC_TIMEOUT_MS)) == 0) { - dev_err(dev, "Timed out initializing SMC"); - return -ETIMEDOUT; - } - - if (smc->boot_stage != APPLE_SMC_INITIALIZED) { - dev_err(dev, "SMC failed to boot successfully, boot stage=%d\n", smc->boot_stage); - return -EIO; - } - dev_set_drvdata(&pdev->dev, smc); BLOCKING_INIT_NOTIFIER_HEAD(&smc->event_handlers); - ret = apple_smc_read(smc, SMC_KEY(#KEY), &count, sizeof(count)); - if (ret >= 0 && ret != sizeof(count)) { - count = 0; - ret = -EIO; - } - if (ret < 0) - return dev_err_probe(smc->dev, ret, "Failed to get key count"); - smc->key_count = be32_to_cpu(count); + ret = apple_smc_handshake(smc); + if (ret) + return ret; /* Notifications, GPIO and power management are not qualified on T8152. */ if (of_device_is_compatible(dev->of_node, "apple,t8152-smc")) @@ -651,6 +719,19 @@ static int apple_smc_probe(struct platform_device *pdev) if (ret) return ret; + /* + * Without the registration (no sleep controller with last stops for this kernel, + * or no room in its table) the noirq suspend commits, as on every other SoC. + */ + if (hw && hw->commit_at_stop && + !apple_sleep_coproc_register_first(dev, APPLE_SLEEP_FIRST_SMC, apple_smc_last_stop, + smc)) { + smc->commit_at_stop = true; + ret = devm_add_action_or_reset(dev, apple_smc_last_stop_unregister, smc); + if (ret) + return ret; + } + /* Enable notifications */ apple_smc_write_flag(smc, SMC_KEY(NTAP), true); ret = devm_add_action_or_reset(dev, apple_smc_disable_notifications, smc); @@ -678,9 +759,101 @@ static int apple_smc_probe(struct platform_device *pdev) ARRAY_SIZE(apple_smc_t6050_described_devs)); } +static int apple_smc_restore(struct device *dev); + +static int apple_smc_freeze(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + /* Only the image-loading kernel, right before it enters the image. */ + if (pm_hibernate_is_loading() && apple_rtkit_is_running(smc->rtk)) { + int ret; + + apple_smc_write_flag(smc, SMC_KEY(NTAP), false); + ret = apple_rtkit_quiesce(smc->rtk); + if (ret) { + /* + * Failing here would abandon the image, whose signature is + * already gone, and leave this kernel without a session: enter + * the image anyway, its restore tries a new session. + */ + dev_err(dev, "could not quiesce the SMC for the image: %d\n", ret); + return 0; + } + apple_smc_handed_over = true; + dev_info(dev, "SMC quiesced for the restored kernel\n"); + return 0; + } + + /* + * Hibernating: the SMC keeps running, and with it thermal and power + * management, while the image is written and until the power-off + * (.poweroff shares this). Nothing is saved: the boot that loads the + * image restarts the SMC, and apple_smc_restore() starts a new session. + */ + return 0; +} + +static int apple_smc_thaw(struct device *dev) +{ + /* The image could not be entered after the hand-over: start a new session. */ + if (apple_smc_handed_over) { + apple_smc_handed_over = false; + return apple_smc_restore(dev); + } + + /* The image was written or abandoned: .freeze left the session running. */ + return 0; +} + +static int apple_smc_restore(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + int ret; + + /* + * The boot that loaded the image restarted the SMC. The RTKit session + * in the image belongs to a firmware instance that no longer runs, and + * its power state still reads running, which apple_rtkit_wake() + * refuses. Reset the transport to the state probe starts from (power + * state off, no endpoints, mailbox receive restarted) and repeat + * probe's handshake with the new instance. + */ + ret = apple_rtkit_reinit(smc->rtk); + if (ret) { + dev_err(dev, "Failed to reinit RTKit on restore: %d\n", ret); + return ret; + } + + smc->boot_stage = APPLE_SMC_BOOTING; + + ret = apple_rtkit_wake(smc->rtk); + if (ret) { + dev_err(dev, "Failed to wake up SMC on restore: %d\n", ret); + return ret; + } + + reinit_completion(&smc->init_done); + reinit_completion(&smc->cmd_done); + + ret = apple_smc_handshake(smc); + if (ret) + return ret; + + /* + * Re-enable the firmware-side notification flag. The event_handlers + * notifier chain and the sub-device registrations on it survive in the + * restored image, so only the firmware flag has to be re-armed. + */ + apple_smc_write_flag(smc, SMC_KEY(NTAP), true); + + return 0; +} + static const struct of_device_id apple_smc_of_match[] = { { .compatible = "apple,t8152-smc" }, { .compatible = "apple,t6050-smc" }, + { .compatible = "apple,t6041-smc", .data = &apple_smc_t6041_hw }, { .compatible = "apple,t8103-smc" }, { .compatible = "apple,smc" }, {}, @@ -690,8 +863,8 @@ MODULE_DEVICE_TABLE(of, apple_smc_of_match); /* * The SMC keeps running through a firmware-driven suspend. It is told first and * stops last, and on a wake it hears first. In order, measured on T8140 and - * unverified on other SoCs: MBSE = slpw before any other coprocessor is told; - * MBSE = slpa and the AP power notice after all the others; on a wake the notice, + * unverified on other SoCs: MBSE = slpw, MBSE = slpa and the AP power notice after + * all the other coprocessors (slpw goes ahead of them); on a wake the notice, * MBSE and MBSW read, MBSE = waka. The noirq phases give that order whatever the * device order is, so these commands poll. What the values mean is inferred * from their letters. @@ -702,15 +875,22 @@ MODULE_DEVICE_TABLE(of, apple_smc_of_match); * The debug link's cut, on a debug cable only, makes the SMC see a cable pulled and plugged * back. With the cut after slpw, the SMC woke the AP right after the commit, for USB-C_plug * or acdetach, in every such sleep (J700, 223 of 223), even with its outbox quiet for 500 ms - * before the notice. A sleep that cuts the link warns the SMC only after the cut. + * before the notice. So the warning comes after the cut. + * + * A T6041 SMC takes neither the warning nor the commit back: see apple_smc_sleep_enter() and + * apple_smc_sleep_commits(). After the noirq suspend a sleep can still be refused: by a wake + * event, by a CPU that does not park, by the sleep controller's checks at its commit, by the + * enclave before its sleep request. So where the controller stops coprocessors at its commit, + * with interrupts off, one CPU up and every refusal behind it, the whole sequence is sent + * from there, ahead of the other coprocessors' stops: apple_smc_last_stop(). */ /* The gaps macOS leaves before the AP idle notice and after it. */ #define SMC_SLEEP_ARM_MS 6 #define SMC_SLEEP_IDLE_MS 30 /* - * Between a late warning (a sleep that cut the debug link) and slpa. macOS's shortest gap - * was 770 ms; the warning at .prepare leaves 150-500 ms, and those sleeps run to the alarm. + * Between the warning and slpa. Sleeps with 150-500 ms between them ran to the alarm; + * 770 ms is known to work. */ #define SMC_SLEEP_WARN_MS 200 @@ -720,9 +900,15 @@ MODULE_DEVICE_TABLE(of, apple_smc_of_match); static int apple_smc_ap_power(struct apple_smc *smc, bool on) { + unsigned int us; + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) KUNIT_STATIC_STUB_REDIRECT(apple_smc_ap_power, smc, on); + /* With interrupts off there is only the idle notice: see apple_smc_last_stop(). */ + if (smc->atomic_mode) + return on ? -EINVAL : apple_rtkit_ap_idle_polled(smc->rtk, &us); + return smc->polled ? apple_rtkit_ap_power_noirq(smc->rtk, on) : apple_rtkit_ap_power(smc->rtk, on); } @@ -750,6 +936,36 @@ static void apple_smc_write_mbse(struct apple_smc *smc, smc_key value) dev_err(smc->dev, "Failed to issue MBSE = %p4ch: %d\n", &value, ret); } +/* + * SMC key pmVC, the request for the VDD_CIO rail: the enable mask | 0x100 + * (0x101 on the J616s), written while an ATC port's always-on side is powered, also at sleep. Linux + * never asks for the rail, which is power-gated otherwise (ADT vdd-cio-rail-pg), and a rail + * the SMC drops during the sleep transition races the sleep engine's hand-over of the ATC + * ports. + */ +#define SMC_PMVC_REQUEST 0x101 + +/** + * apple_smc_request_pmvc() - ask the SMC to keep the VDD_CIO rail up + * @smc: the SMC + * + * Writes pmVC at the moment the caller picks; it does not hook the SMC's own suspend + * callbacks, so a sleep controller that has to issue it ahead of the device suspend + * phase keeps that order. + * + * Return: 0 on success, a negative errno otherwise. + */ +int apple_smc_request_pmvc(struct apple_smc *smc) +{ + int ret; + + ret = apple_smc_write_u32(smc, __SMC_KEY('p', 'm', 'V', 'C'), SMC_PMVC_REQUEST); + if (ret < 0) + dev_warn(smc->dev, "pmVC: write %#x failed: %d\n", SMC_PMVC_REQUEST, ret); + return ret; +} +EXPORT_SYMBOL(apple_smc_request_pmvc); + static void apple_smc_sleep_warn(struct apple_smc *smc) { apple_smc_write_mbse(smc, SMC_KEY(slpw)); @@ -780,10 +996,26 @@ static int apple_smc_notice_idle(struct apple_smc *smc) return ret; } +/* + * WKTP, one byte, the value for power state 0. It goes first at the sleep request, + * ahead of the other sleep keys ('slpa' among them) and the AP idle notice. Only for + * an SMC known to take it: T6041's. + */ +#define SMC_WKTP_SLEEP 2 +/* And once a wake is a full one: see apple_smc_wake_full(). */ +#define SMC_WKTP_AWAKE 0 + static int apple_smc_sleep_commit(struct apple_smc *smc) { int ret; + /* Whatever is written here, the SMC runs the wake after this silent until it is told. */ + smc->wake_silent = true; + if (smc->wktp) { + ret = apple_smc_write_u8(smc, SMC_KEY(WKTP), SMC_WKTP_SLEEP); + if (ret) + dev_err(smc->dev, "Failed to write WKTP = %u: %d\n", SMC_WKTP_SLEEP, ret); + } apple_smc_write_mbse(smc, SMC_KEY(slpa)); /* Device interrupts are off here. */ mdelay(SMC_SLEEP_ARM_MS); @@ -795,6 +1027,26 @@ static int apple_smc_sleep_commit(struct apple_smc *smc) return ret; } +/* + * WKTP once a wake is a full one: 0, where a wake that stays dark keeps the key nonzero, + * Without the 0 the SMC treats all the time awake as a power nap, whether + * the sleep wrote its 2 or not: it keeps its own fan policy against a manual fan request + * (the mode request times out), and under thermal load it requests a sleep once a second. + */ +static void apple_smc_wake_full(struct apple_smc *smc) +{ + int ret; + + if (!smc->wake_silent) + return; + smc->wake_silent = false; + if (!smc->wktp) + return; + ret = apple_smc_write_u8(smc, SMC_KEY(WKTP), SMC_WKTP_AWAKE); + if (ret) + dev_err(smc->dev, "Failed to write WKTP = %u: %d\n", SMC_WKTP_AWAKE, ret); +} + static int apple_smc_wake_reason_names(const struct apple_smc *smc, u64 record, char *buf, size_t size) { @@ -902,6 +1154,9 @@ static void apple_smc_sleep_wake(struct apple_smc *smc) apple_smc_lid_wake(smc); if (apple_smc_port_reasons_only(smc, smc->wake_record)) apple_sleep_debug_link_cut_wake(); + /* A dark wake sleeps again, and its commit writes the key anyway. */ + if (!pm_dark_wake()) + apple_smc_wake_full(smc); /* A dark wake isn't one: with wakeup_count in use, it would abort the sleep after it. */ if (smc->wake_record && !pm_dark_wake()) pm_wakeup_event(smc->dev, 0); @@ -913,29 +1168,35 @@ static int apple_smc_prepare(struct device *dev) if (!pm_suspend_via_firmware()) return 0; - if (apple_sleep_debug_link_up()) - smc->sleep_warn_late = true; - else - apple_smc_sleep_warn(smc); + /* Warned with the commit, not here: see apple_smc_sleep_enter(). */ + smc->sleep_warn_late = true; return 0; } /* Warned, and a refusal came before the notice: tell it the sleep is off. */ -static void apple_smc_complete(struct device *dev) +static void apple_smc_sleep_end(struct apple_smc *smc) { - struct apple_smc *smc = dev_get_drvdata(dev); - + if (smc->commit_at_stop) + apple_sleep_coproc_set_ready(smc->dev, false); smc->sleep_warn_late = false; if (smc->sleep_warned) { apple_smc_write_mbse(smc, SMC_KEY(waka)); smc->sleep_warned = false; } + /* A commit the SMC refused, or dark wakes that ended in a refusal. */ + apple_smc_wake_full(smc); +} + +static void apple_smc_complete(struct device *dev) +{ + apple_smc_sleep_end(dev_get_drvdata(dev)); } /* * For at least @min_ms, then until the outbox is quiet for @quiet_ms. The events go to their - * handlers as usual, through the RTKit worker. + * handlers as usual, through the RTKit worker. With interrupts off, in atomic mode, that + * worker runs them once the sleep is over. */ static void apple_smc_outbox_poll(struct apple_smc *smc, const char *after, unsigned int min_ms, unsigned int quiet_ms) @@ -955,18 +1216,147 @@ static void apple_smc_outbox_poll(struct apple_smc *smc, const char *after, } else { quiet++; } - usleep_range(1000, 1200); + if (smc->atomic_mode) + mdelay(1); + else + usleep_range(1000, 1200); } /* A debug cable only. */ dev_info(smc->dev, "%s: %u messages in %u ms, the last at %u ms%s\n", after, count, ms, last, ms >= SMC_OUTBOX_SETTLE_MS ? ", still busy" : ""); } -/* The noirq suspend's part, polled: the cut, a late warning, the commit's notices. */ +/* + * Whether the sleep controller's last stops send this sleep's commit: the SMC is registered + * for one, and the controller runs them for this sleep. + */ +static bool apple_smc_commits_at_stop(struct apple_smc *smc) +{ + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_commits_at_stop, smc); + + return smc->commit_at_stop && apple_sleep_commit_stops_coprocs(); +} + +static int apple_smc_stop_write(struct apple_smc *smc, smc_key key, const void *buf, size_t size) +{ + int ret = apple_smc_write_atomic(smc, key, buf, size); + + if (ret < 0) + dev_err(smc->dev, "Failed to write %p4ch at the last stop: %d\n", &key, ret); + + return ret < 0 ? ret : 0; +} + +/* + * The warning and the commit as one of the sleep controller's last stops: interrupts are + * off, one CPU is up, the enclave has its sleep request and nothing can refuse any more. So + * an SMC that got any of this also gets the power loss that brings it out of its sleep + * state. The order and the gaps are the noirq suspend's: apple_smc_sleep_enter() and + * apple_smc_sleep_commit(). + * + * Nothing sleeps here and the RTKit worker does not run, so the writes are the atomic ones + * and the idle notice's answer is polled for. Every other user of the SMC is suspended, and + * the mutex is not taken. An event the SMC sends meanwhile is taken from its outbox and + * reaches its handler after the wake. + * + * A last stop cannot refuse, and the sleep goes on whatever the SMC answers. So does this: + * a step that fails is logged and the rest is sent all the same, since an SMC that took + * the warning leaves its sleep state only through the rest of the sequence and the power + * loss. The one exception is an SMC that does not answer at all: a write waits up to + * SMC_ATOMIC_TIMEOUT_MS for its answer, so after the first timeout the rest is left out + * instead of waiting that long again for each step. The wake sends its part, as after any + * sleep, once this has started. Returns the first failure. + */ +static bool apple_smc_stop_dead(int ret) +{ + /* No answer within SMC_ATOMIC_TIMEOUT_MS, or the one before is still missing. */ + return ret == -ETIMEDOUT || ret == -EBUSY; +} + +static int apple_smc_last_stop(void *data) +{ + const u32 slpw = SMC_KEY(slpw), slpa = SMC_KEY(slpa); + const u8 wktp = SMC_WKTP_SLEEP; + struct apple_smc *smc = data; + int ret, err; + + /* Not left for here: the noirq suspend sent it, or this suspend is not a sleep. */ + if (!smc->sleep_warn_late) + return 0; + smc->sleep_warn_late = false; + + scoped_guard(spinlock_irqsave, &smc->lock) + smc->atomic_mode = true; + + err = apple_smc_stop_write(smc, SMC_KEY(MBSE), &slpw, sizeof(slpw)); + smc->sleep_warned = true; + if (apple_smc_stop_dead(err)) + goto dead; + apple_smc_outbox_poll(smc, "sleep warning at the last stop", SMC_SLEEP_WARN_MS, 0); + + /* As in apple_smc_sleep_commit(). */ + smc->wake_silent = true; + if (smc->wktp) { + ret = apple_smc_stop_write(smc, SMC_KEY(WKTP), &wktp, sizeof(wktp)); + err = err ?: ret; + if (apple_smc_stop_dead(ret)) + goto dead; + } + ret = apple_smc_stop_write(smc, SMC_KEY(MBSE), &slpa, sizeof(slpa)); + err = err ?: ret; + if (apple_smc_stop_dead(ret)) + goto dead; + mdelay(SMC_SLEEP_ARM_MS); + + /* Not taken back if the notice fails: the sleep goes on, and its wake sends the other. */ + smc->ap_notified = true; + ret = apple_smc_ap_power(smc, false); + if (ret) + dev_err(smc->dev, "SMC did not take the sleep notice at the last stop: %d\n", ret); + else + mdelay(SMC_SLEEP_IDLE_MS); + err = err ?: ret; + goto out; + +dead: + /* + * An SMC that answers nothing for SMC_ATOMIC_TIMEOUT_MS takes nothing more: each further + * write would wait that long again, and again for the answer still missing, all with + * interrupts off. The sleep goes on without the rest. + */ + dev_err(smc->dev, "SMC does not answer at the last stop, the rest is left out\n"); + err = err ?: -ETIMEDOUT; +out: + /* + * The wake's commands are the usual ones. An answer that never came is given up: if + * it comes after all, it fails the sequence check of the command it lands on. + */ + scoped_guard(spinlock_irqsave, &smc->lock) { + smc->atomic_pending = false; + smc->atomic_mode = false; + } + + return err; +} + +/* + * The noirq suspend's part, polled: the cut, the warning, the commit's notices. Where the + * sleep controller's last stops send the warning and the commit, only the cut is left here: + * it sleeps, and it has to come before the warning. + * + * The warning is not sent earlier, although the SMC could take it then. A T6041 SMC that + * was warned and then told that the sleep was off, with no commit between, stopped running + * the fans (0 rpm at a die temperature of 110 C) and refused a fan request, until a later + * sleep had completed: nothing sent to it takes the warning back. So it goes where the + * commit goes, after every device has had its chance to refuse. + */ static int apple_smc_sleep_enter(struct apple_smc *smc) { if (apple_sleep_debug_link_cut()) apple_smc_outbox_poll(smc, "debug link cut", 0, SMC_OUTBOX_QUIET_MS); + if (apple_smc_commits_at_stop(smc)) + return 0; if (smc->sleep_warn_late) { smc->sleep_warn_late = false; apple_smc_sleep_warn(smc); @@ -976,13 +1366,37 @@ static int apple_smc_sleep_enter(struct apple_smc *smc) return apple_smc_sleep_commit(smc); } -static int apple_smc_suspend_noirq(struct device *dev) +/* + * Whether this suspend goes on to the SoC's power loss. Only then is the SMC given its sleep + * commit, which nothing sent to it takes back: a T6041 SMC that took the commit and then saw + * no power loss kept every sensor report at its last value until a later sleep had completed, + * although it got the notices of a wake as after any sleep. So the commit is among the stops + * that only a power cycle undoes, and a dry run, which ends short of the power loss by + * design, leaves it out. + */ +static bool apple_smc_sleep_commits(void) +{ + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_sleep_commits); + + return apple_sleep_power_loss(); +} + +static int apple_smc_sleep_noirq(struct apple_smc *smc) { - struct apple_smc *smc = dev_get_drvdata(dev); int ret; if (!smc->sleep_warned && !smc->sleep_warn_late) return 0; + /* + * Nothing of the SMC's is in the way of the sleep controller's stops, in a dry run's + * checks as in a sleep. Its last stop runs only where a sleep is committed. + */ + if (smc->commit_at_stop) + apple_sleep_coproc_set_ready(smc->dev, true); + /* A sleep that stops short of the commit sends the SMC nothing. */ + if (!apple_smc_sleep_commits()) + return 0; apple_smc_set_polled(smc, true); ret = apple_smc_sleep_enter(smc); @@ -991,22 +1405,49 @@ static int apple_smc_suspend_noirq(struct device *dev) return ret; } -static int apple_smc_resume_noirq(struct device *dev) +static int apple_smc_suspend_noirq(struct device *dev) { - struct apple_smc *smc = dev_get_drvdata(dev); + return apple_smc_sleep_noirq(dev_get_drvdata(dev)); +} - if (smc->ap_notified) { - apple_smc_set_polled(smc, true); - apple_smc_sleep_wake(smc); - apple_smc_set_polled(smc, false); - } +static void apple_smc_wake_noirq(struct apple_smc *smc) +{ + if (!smc->ap_notified) + return; + + /* + * Committed, and no power loss followed: a check that comes after this device's + * suspend refused the sleep, or a wake event got in first. The SMC gets the notices + * of a wake all the same, which is all there is to send, but it leaves its sleep + * state only through a power loss: see apple_smc_sleep_commits(). A commit sent from + * the sleep controller's last stop has every refusal behind it. + */ + if (!pm_resume_via_firmware()) + dev_warn(smc->dev, "sleep refused after the SMC's commit: its sensor reports may stand still until a sleep completes\n"); + + apple_smc_set_polled(smc, true); + apple_smc_sleep_wake(smc); + apple_smc_set_polled(smc, false); +} + +static int apple_smc_resume_noirq(struct device *dev) +{ + apple_smc_wake_noirq(dev_get_drvdata(dev)); return 0; } +/* + * Suspend-to-RAM commits in the noirq phase or from the sleep controller's last stop; + * hibernation restarts the session in the normal one. + */ static const struct dev_pm_ops apple_smc_pm_ops = { .prepare = pm_sleep_ptr(apple_smc_prepare), .complete = pm_sleep_ptr(apple_smc_complete), + .freeze = pm_sleep_ptr(apple_smc_freeze), + .thaw = pm_sleep_ptr(apple_smc_thaw), + .poweroff = pm_sleep_ptr(apple_smc_freeze), + .restore = pm_sleep_ptr(apple_smc_restore), NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_smc_suspend_noirq, apple_smc_resume_noirq) }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index b61a51dbbce1a..1311738f3ae6a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5274,11 +5274,6 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, * in suspended state */ brcmf_link_down(vif, WLAN_REASON_UNSPECIFIED, true); - /* Make sure WPA_Supplicant receives all the event - * generated due to DISASSOC call to the fw to keep - * the state fw and WPA_Supplicant state consistent - */ - brcmf_delay(500); } /* Configure MPC */ brcmf_set_mpc(ifp, 1); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 2bfee245f4331..b57310bc2008b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -10,11 +10,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -3479,11 +3481,75 @@ static int brcmf_pcie_pm_enter_D3(struct device *dev) } +static struct pci_driver brcmf_pciedrvr; +static struct workqueue_struct *brcmf_pcie_restart_wq; + +struct brcmf_pcie_restart { + struct work_struct work; + struct pci_dev *pdev; +}; + +static void brcmf_pcie_restart_work(struct work_struct *work) +{ + struct brcmf_pcie_restart *restart = + container_of(work, struct brcmf_pcie_restart, work); + struct pci_dev *pdev = restart->pdev; + int err; + + /* The resume callback that queued this still holds the lock; so does an unbind. */ + device_lock(&pdev->dev); + if (pdev->dev.driver == &brcmf_pciedrvr.driver && dev_get_drvdata(&pdev->dev)) { + brcmf_pcie_remove(pdev); + err = brcmf_pcie_probe(pdev, NULL); + if (err) + __brcmf_err(NULL, __func__, "probe after resume failed, err=%d\n", + err); + } + device_unlock(&pdev->dev); + pci_dev_put(pdev); + kfree(restart); +} + +/* Whether the remove and probe of a function back from a power loss were queued. */ +static bool brcmf_pcie_restart_queue(struct pci_dev *pdev) +{ + struct brcmf_pcie_restart *restart; + + if (!brcmf_pcie_restart_wq) + return false; + restart = kzalloc(sizeof(*restart), GFP_KERNEL); + if (!restart) + return false; + INIT_WORK(&restart->work, brcmf_pcie_restart_work); + restart->pdev = pci_dev_get(pdev); + queue_work(brcmf_pcie_restart_wq, &restart->work); + return true; +} + +/* The next sleep finds the function probed again, not in the middle of it. */ +static int brcmf_pcie_restart_pm(struct notifier_block *nb, unsigned long action, void *data) +{ + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + if (brcmf_pcie_restart_wq) + flush_workqueue(brcmf_pcie_restart_wq); + break; + } + return NOTIFY_DONE; +} + +static struct notifier_block brcmf_pcie_restart_nb = { + .notifier_call = brcmf_pcie_restart_pm, +}; + static int brcmf_pcie_pm_leave_D3(struct device *dev) { struct brcmf_pciedev_info *devinfo; struct brcmf_bus *bus; struct pci_dev *pdev; + u32 intmask; int err; err = apple_radio_sleep_endpoint_resume(dev); @@ -3509,8 +3575,19 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) return 0; } + /* + * ~0 is unbacked MMIO (link still down after the Apple RC power + * cycle), not a live firmware. The hot-resume path would issue more + * mailbox MMIO and wedge. Fall through to remove+probe. + */ + intmask = brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->intmask); + if (intmask == ~0u) { + brcmf_err(bus, "INTMASK read ~0 on resume; reloading firmware\n"); + goto cleanup; + } + /* Check if device is still up and running, if so we are ready */ - if (brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->intmask) != 0) { + if (intmask != 0) { brcmf_dbg(PCIE, "Try to wakeup device....\n"); /* Set the device up, so we can write the MB data message in ring mode */ devinfo->state = BRCMFMAC_PCIE_STATE_UP; @@ -3537,6 +3614,15 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) brcmf_chip_detach(devinfo->ci); devinfo->ci = NULL; pdev = devinfo->pdev; + /* + * A function that lost its power in a firmware suspend, or in the boot that loaded a + * hibernation image, starts over. Taking the interfaces and the wiphy down and the + * firmware download take about 600 ms, and everything that follows this device in + * the resume waits for it, so do it right behind the resume. + */ + if ((pm_resume_via_firmware() || pm_hibernate_is_restored()) && + brcmf_pcie_restart_queue(pdev)) + return 0; brcmf_pcie_remove(pdev); err = brcmf_pcie_probe(pdev, NULL); @@ -3627,13 +3713,45 @@ static struct pci_driver brcmf_pciedrvr = { int brcmf_pcie_register(void) { + int ret; + brcmf_dbg(PCIE, "Enter\n"); - return pci_register_driver(&brcmf_pciedrvr); +#ifdef CONFIG_PM + /* + * Without it the restart stays in the resume callback. Freezable: the work then + * starts once the resume of every device is over. Run during it, the removal of + * the wiphy only moves the wait to the wiphy's own resume. + */ + brcmf_pcie_restart_wq = alloc_ordered_workqueue("brcmf_pcie_restart", WQ_FREEZABLE); + if (brcmf_pcie_restart_wq) + register_pm_notifier(&brcmf_pcie_restart_nb); +#endif + ret = pci_register_driver(&brcmf_pciedrvr); +#ifdef CONFIG_PM + if (ret && brcmf_pcie_restart_wq) { + unregister_pm_notifier(&brcmf_pcie_restart_nb); + destroy_workqueue(brcmf_pcie_restart_wq); + brcmf_pcie_restart_wq = NULL; + } +#endif + return ret; } void brcmf_pcie_exit(void) { brcmf_dbg(PCIE, "Enter\n"); +#ifdef CONFIG_PM + if (brcmf_pcie_restart_wq) { + unregister_pm_notifier(&brcmf_pcie_restart_nb); + flush_workqueue(brcmf_pcie_restart_wq); + } +#endif pci_unregister_driver(&brcmf_pciedrvr); +#ifdef CONFIG_PM + if (brcmf_pcie_restart_wq) { + destroy_workqueue(brcmf_pcie_restart_wq); + brcmf_pcie_restart_wq = NULL; + } +#endif } diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 88293eaf15367..0b1182064f97c 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -215,13 +215,17 @@ struct apple_nvme_hw { bool needs_ioq_registers; /* T8132 needs a PMGR reset when starting a stopped coprocessor. */ bool cold_reset; - /* Firmware-driven suspend has been traced on this controller. */ - bool sleep_traced; - /* The last endpoint map reply from that trace */ + /* + * The sleep controller takes this ANS through a firmware-driven suspend: + * flush, quiesce and shutdown notification at the device suspend, the + * polled idle at the commit and a warm restart after the wake. + */ + bool sleep_coproc; + /* The last endpoint map reply of that warm restart; 0: the whole map again. */ u32 sleep_epmap_last_mask; /* Higher namespaces are the firmware's, and a Flush to one faults the controller. */ u32 sleep_flush_max_nsid; - /* CC.SHN for a sleep, as read back from the OS the sleep was traced on. */ + /* CC.SHN the controller wants for a sleep. */ u32 sleep_shn; /* The ANS is never reset through PMGR here, so no reset is named. */ bool reset_optional; @@ -2399,6 +2403,9 @@ static void apple_nvme_free_rtkit(void *data) apple_nvme_set_reset_ready(anv, false); guard(mutex)(&anv->disable_lock); + /* A failed hibernation restore leaves no session behind. */ + if (!anv->rtk) + return; if (anv->hw->needs_ioq_registers) apple_rtkit_free_retaining_buffers(anv->rtk); else @@ -2428,6 +2435,24 @@ static void apple_nvme_flush_work(struct work_struct *work) } } +/* + * What a new RTKit object is told about this controller: at probe, and when a + * hibernation restore replaces the object with one for the session the boot + * loader started. + */ +static void apple_nvme_rtkit_configure(struct apple_nvme *anv) +{ + /* + * The T6050 ANS firmware handed over by iBoot can take about 3 s to + * answer the power-state and endpoint-map handshake. + */ + if (of_device_is_compatible(anv->dev->of_node, "apple,t6050-nvme-ans2")) + apple_rtkit_set_handshake_timeout(anv->rtk, + APPLE_NVME_T6050_HANDSHAKE_TIMEOUT_MS); + if (anv->hw->sleep_coproc) + apple_rtkit_set_warm_epmap_mask(anv->rtk, anv->hw->sleep_epmap_last_mask); +} + static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -2520,7 +2545,7 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) /* * Newer ANS firmware moves the linear submission queue registers into * the NVMMU block and ends the "nvme" block before them (T6040, and - * T8142 booting the macOS 27.0 stub: 0x10000, ADT ans reg[9]). A + * T8142 with the 27.0 firmware stub: window of 0x10000). A * window that does not reach the doorbells means that layout. */ anv->lsq_in_nvmmu = anv->hw->lsq_in_nvmmu || @@ -2627,13 +2652,7 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) goto out_detach_genpd; } - /* - * The T6050 ANS firmware handed over by iBoot can take about 3 s to - * answer the power-state and endpoint-map handshake. - */ - if (of_device_is_compatible(dev->of_node, "apple,t6050-nvme-ans2")) - apple_rtkit_set_handshake_timeout(anv->rtk, - APPLE_NVME_T6050_HANDSHAKE_TIMEOUT_MS); + apple_nvme_rtkit_configure(anv); ret = devm_add_action_or_reset(dev, apple_nvme_free_rtkit, anv); if (ret) @@ -2742,7 +2761,7 @@ static int apple_nvme_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush_work); - if (anv->hw->sleep_traced) { + if (anv->hw->sleep_coproc) { anv->sleep_registered = !apple_sleep_coproc_register(anv->dev, anv->mmio_coproc, true, apple_nvme_sleep_last_stop, anv); @@ -2752,7 +2771,6 @@ static int apple_nvme_probe(struct platform_device *pdev) apple_sleep_coproc_unregister(anv->dev); anv->sleep_registered = false; } - apple_rtkit_set_warm_epmap_mask(anv->rtk, anv->hw->sleep_epmap_last_mask); } apple_nvme_set_reset_ready(anv, true); @@ -2840,20 +2858,20 @@ static void apple_nvme_shutdown(struct platform_device *pdev) } } -/* macOS sees the firmware report on 68 ms after HELLO. Each try waits RTKit's timeout again. */ +/* The firmware reports on about 68 ms after HELLO. Each try waits RTKit's timeout again. */ #define APPLE_NVME_SLEEP_ON_TRIES 15 /* * After a real sleep ANS has lost power, and its resident firmware restarts once the run bit - * is set. Measured on macOS under aura, the same in five runs: HELLO first and unprompted, + * is set. Measured under aura, the same in five runs: HELLO first and unprompted, * then the endpoint map, the last part answered with endpoint 0x20 only. No endpoint is * started and no buffer asked for. About 68 ms later the firmware reports on, and the AP state * follows, with the controller's registers reading 0. A refused suspend never told the * firmware anything, and takes a reset. * - * The run bit goes in the early phase, so HELLO comes within milliseconds of the wake as on - * macOS. The wait for on doesn't: macOS rings the SEP a few ms after ANS's HELLO, before ANS - * reports on, so the reset work waits, after the SEP's own resume has had its turn. + * The run bit goes in the early phase, so HELLO comes within milliseconds of the wake. The + * wait for on doesn't: the SEP is rung a few ms after ANS's HELLO, before ANS reports on, + * so the reset work waits, after the SEP's own resume has had its turn. */ static int apple_nvme_resume_early(struct device *dev) { @@ -2869,7 +2887,7 @@ static int apple_nvme_resume_early(struct device *dev) apple_sleep_coproc_set_ready(dev, false); /* Its link doesn't come up with the pins as the sleep left them. */ apple_sleep_restore_pins(); - /* macOS writes the SART's entries back again right before the run bit. */ + /* The SART's entries go back in again right before the run bit. */ if (anv->sart) apple_sart_replay(anv->sart); @@ -3006,12 +3024,12 @@ static bool apple_nvme_sleep_wanted(struct apple_nvme *anv) } /* - * As macOS sends it, from its power-state change during the device suspend: the shutdown - * notification, waited for with interrupts on for as long as macOS allows (35 s). The - * controller can take seconds to save its context (1.7 s in one macOS sleep). A sleep - * refused later finds it shut down, and resume's controller reset takes it back. + * Sent from the device suspend: the shutdown notification, waited for with interrupts on + * for up to 35 s. The controller can take seconds to save its context (1.7 s in one + * sleep). A sleep refused later finds it shut down, and resume's controller reset takes it + * back. */ -#define APPLE_ANS_SLEEP_SHN_MACOS_TIMEOUT_US (35 * USEC_PER_SEC) +#define APPLE_ANS_SLEEP_SHN_TIMEOUT_US (35 * USEC_PER_SEC) static int apple_nvme_sleep_send_shn(struct apple_nvme *anv) { @@ -3025,7 +3043,7 @@ static int apple_nvme_sleep_send_shn(struct apple_nvme *anv) writel(anv->ctrl.ctrl_config, anv->mmio_nvme + NVME_REG_CC); ret = readl_poll_timeout(anv->mmio_nvme + NVME_REG_CSTS, csts, (csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT, - 1000, APPLE_ANS_SLEEP_SHN_MACOS_TIMEOUT_US); + 1000, APPLE_ANS_SLEEP_SHN_TIMEOUT_US); us = ktime_us_delta(ktime_get(), start); if (ret) dev_err(anv->dev, "shutdown notification failed after %lld ms: CSTS %#x\n", @@ -3107,8 +3125,122 @@ static int apple_nvme_suspend(struct device *dev) return ret; } +/* + * Hibernation restore on post-M4. The machine cold-booted since .freeze: + * the boot firmware restarted ANS, the boot stages kept it running, and the + * resume kernel adopted that session to read the image (its own queues, and + * its .freeze cleared CC.EN before jumping here). anv->rtk therefore + * describes a firmware instance that no longer exists, and the live session + * on the mailbox has never been adopted by this kernel. Unlike + * apple_nvme_can_adopt_rtkit(), accept a disabled controller: it was just + * disabled below, and RUN + BOOT_STATUS_OK + !CFS are what survive. + */ +static bool apple_nvme_can_adopt_restored_rtkit(struct apple_nvme *anv) +{ + u32 csts; + + if (!(readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & + APPLE_ANS_COPROC_CPU_CONTROL_RUN) || + readl(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS) != APPLE_ANS_BOOT_STATUS_OK) + return false; + csts = readl(anv->mmio_nvme + NVME_REG_CSTS); + if (csts & NVME_CSTS_CFS) + return false; + return !(readl(anv->mmio_nvme + NVME_REG_CC) & NVME_CC_ENABLE) && + !(csts & NVME_CSTS_RDY); +} + +static int apple_nvme_restore_post_m4(struct apple_nvme *anv) +{ + struct apple_rtkit *rtk; + int ret; + + guard(mutex)(&anv->disable_lock); + if (READ_ONCE(anv->quarantined)) + return -EIO; + dev_dbg(anv->dev, + "hibernation restore: CPU_CONTROL=%#x BOOT_STATUS=%#x CC=%#x CSTS=%#x\n", + readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL), + readl(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS), + readl(anv->mmio_nvme + NVME_REG_CC), + readl(anv->mmio_nvme + NVME_REG_CSTS)); + + /* + * Stop the controller first: until CC.EN is clear the firmware may + * still fetch from the resume kernel's queues, whose memory is now + * this kernel's. This is the owned-session reset path. + */ + ret = apple_nvme_disable_locked(anv, false); + if (ret) + return ret; + + /* + * Our firmware instance ended with the power-off. No shutdown + * handshake: the new instance would receive it. Shared buffers are + * retained as in apple_nvme_free_rtkit(). + */ + apple_nvme_set_reset_ready(anv, false); + apple_rtkit_free_retaining_buffers(anv->rtk); + anv->rtk = NULL; + anv->owns_rtkit = false; + + if (!apple_nvme_can_adopt_restored_rtkit(anv)) { + dev_err(anv->dev, "hibernation restore: no adoptable ANS session\n"); + apple_nvme_quarantine(anv, -EIO); + return -EIO; + } + + /* Publish admission before the initializer enables mailbox RX. */ + anv->inherited_rtkit = true; + rtk = apple_rtkit_init_adopted(anv->dev, anv, NULL, 0, + &apple_nvme_rtkit_ops); + if (IS_ERR(rtk)) { + ret = PTR_ERR(rtk); + dev_err(anv->dev, "hibernation restore: RTKit init failed: %d\n", ret); + apple_nvme_quarantine(anv, ret); + return ret; + } + anv->rtk = rtk; + apple_nvme_rtkit_configure(anv); + ret = apple_rtkit_adopt_running(rtk); + if (ret) { + dev_err(anv->dev, "hibernation restore: RTKit adoption failed: %d\n", ret); + apple_nvme_quarantine(anv, ret); + return ret; + } + /* Owned: reset_work takes the live-session path, as after s2idle. */ + anv->owns_rtkit = true; + apple_nvme_set_reset_ready(anv, true); + dev_info(anv->dev, + "hibernation restore: adopted the boot loader's ANS session\n"); + return 0; +} + +static int apple_nvme_restore(struct device *dev) +{ + struct apple_nvme *anv = dev_get_drvdata(dev); + int ret; + + if (anv->hw->needs_ioq_registers) { + ret = apple_nvme_restore_post_m4(anv); + if (ret) + return ret; + } + return apple_nvme_resume(dev); +} + +/* + * freeze and poweroff disable the controller and keep the session, like + * suspend; thaw (image not written) is a resume. Only restore differs. The + * early resume restarts ANS only after a firmware-driven suspend idled it. + */ static const struct dev_pm_ops apple_nvme_pm_ops = { - SYSTEM_SLEEP_PM_OPS(apple_nvme_suspend, apple_nvme_resume) + .suspend = pm_sleep_ptr(apple_nvme_suspend), + .resume = pm_sleep_ptr(apple_nvme_resume), + .freeze = pm_sleep_ptr(apple_nvme_suspend), + .thaw = pm_sleep_ptr(apple_nvme_resume), + .poweroff = pm_sleep_ptr(apple_nvme_suspend), + .restore = pm_sleep_ptr(apple_nvme_restore), LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_nvme_resume_early) }; @@ -3135,7 +3267,7 @@ static const struct apple_nvme_hw apple_nvme_t8132_hw = { .cold_reset = true, .max_queue_depth = 64, /* T8140's values: T8132's sleep path hasn't been traced. */ - .sleep_traced = true, + .sleep_coproc = true, .sleep_epmap_last_mask = BIT(0), .sleep_flush_max_nsid = 1, .sleep_shn = NVME_CC_SHN_MASK, @@ -3148,7 +3280,7 @@ static const struct apple_nvme_hw apple_nvme_t8140_hw = { .has_separate_nvmmu = true, .needs_ioq_registers = true, .max_queue_depth = 64, - .sleep_traced = true, + .sleep_coproc = true, .sleep_epmap_last_mask = BIT(0), .sleep_flush_max_nsid = 1, .sleep_shn = NVME_CC_SHN_MASK, @@ -3183,6 +3315,26 @@ static const struct apple_nvme_hw apple_nvme_t6040_hw = { .needs_ioq_registers = true, .lsq_in_nvmmu = true, .max_queue_depth = 64, + /* + * The sleep controller takes this ANS through a firmware-driven suspend + * as on T8140: flush, quiesce and the shutdown notification at the + * device suspend, the firmware idled at the commit, its storage link + * quiesced behind it and brought up again on wake, then the SART replay, + * the warm RTKit restart and a controller reset. A sleep taken back + * before the commit has flushed, quiesced and notified too, and resets + * the controller like any refused sleep. + * + * The firmware takes the shutdown notification with both CC.SHN bits set + * and answers with CSTS.SHST complete. It owns every namespace above 1, + * as on T8140, so only namespace 1 is flushed. + * + * After a sleep that cut the power it takes the whole endpoint map + * echoed as the last reply of the warm restart, RTKit's default: + * sleep_epmap_last_mask stays 0. + */ + .sleep_coproc = true, + .sleep_flush_max_nsid = 1, + .sleep_shn = NVME_CC_SHN_MASK, .quirks = NVME_QUIRK_NO_APST, }; diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index b6c3b98af2c9e..4bb33b5623212 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -18,6 +18,7 @@ * Author: Marc Zyngier */ +#include #include #include #include @@ -103,16 +104,16 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PHY_LANE_CFG_REFBUF_OFF BIT(4) #define PHY_LANE_CFG_REFCLKEN (BIT(9) | BIT(10)) #define PHY_LANE_CFG_REFCLKCGEN (BIT(30) | BIT(31)) -/* Bit 31 of the pair: the only one a trace of macOS shows cleared, after PERST# is asserted */ +/* Bit 31 of the pair: the only one that is cleared once PERST# is asserted */ #define PHY_LANE_CFG_REFCLKCGEN_B31 BIT(31) #define PHY_LANE_CTL 0x00004 #define PHY_LANE_CTL_CFGACC BIT(15) #define PORT_LTSSMCTL 0x00080 #define PORT_LTSSMCTL_START BIT(0) -/* T6050: a trace of macOS shows 0x11 written here to send PME_Turn_Off */ +/* T6050: writing 0x11 here sends PME_Turn_Off */ #define PORT_PME_TURN_OFF 0x00088 -/* Reads back 0 within a few reads of the write (the trace reads 0x1, then 0x10) */ +/* Reads back 0 within a few reads of the write (first 0x1, then 0x10) */ #define PORT_PME_TURN_OFF_SEND BIT(0) /* Written with bit 0 and still set afterwards; meaning not known */ #define PORT_PME_TURN_OFF_BIT4 BIT(4) @@ -144,7 +145,7 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PORT_LINKSTS 0x00208 #define PORT_LINKSTS_UP BIT(0) #define PORT_LINKSTS_BUSY BIT(2) -/* In traces of macOS: 0x9b00134d for a port that reached L2, 0x9b001001 for one that did not */ +/* Status word: 0x9b00134d for a port that reached L2, 0x9b001001 for one that did not */ #define PORT_LINKSTS_L2 BIT(6) #define PORT_LINKCMDSTS 0x00210 #define PORT_OUTS_NPREQS 0x00284 @@ -275,6 +276,18 @@ struct hw_info { bool pipe_rst; bool no_port_reset; bool mps_timers; + /* Cold init waits for port RUN before the root-port config tunables. */ + bool tunnel_ready_before_rc; + /* The router acknowledges a tunneled reset by setting ACK_PEND. */ + bool tunnel_ack_on_set; + /* Stop the tunnel for hibernation and start it on thaw and restore. */ + bool tunnel_hibernate; + /* + * Internal ports with their endpoints, which a restored image or a real S2R finds + * reset behind a down link: probe's bring-up runs again, see + * apple_pcie_conventional_host(). + */ + bool conventional_restore; }; static const struct hw_info t8103_hw = { @@ -286,6 +299,7 @@ static const struct hw_info t8103_hw = { .port_rid2sid = PORT_RID2SID, .port_msimap = 0, .max_rid2sid = 64, + .conventional_restore = true, }; static const struct hw_info t602x_hw = { @@ -298,6 +312,7 @@ static const struct hw_info t602x_hw = { .port_msimap = PORT_T602X_MSIMAP, /* 16 on t602x, guess for autodetect on future HW */ .max_rid2sid = 512, + .conventional_restore = true, }; /* @@ -330,6 +345,7 @@ static const struct hw_info t8152_hw = { .max_rid2sid = 512, .refbuf_off = true, .pipe_rst = true, + .conventional_restore = true, }; /* @@ -345,6 +361,7 @@ static const struct hw_info t6050_hw = { .max_rid2sid = 64, .retain_clocks = true, .adopt_link = true, + .conventional_restore = true, }; #define T6050_ROUTE(rid, sid) (PORT_RID2SID_VALID | \ @@ -405,6 +422,16 @@ static const struct hw_info t8132_pciec_hw = { .s2idle_hold_domain = true, }; +/* + * T6040/T6041 (M4 Pro/Max) PCIe-C is the T8132 block with only its own + * register indices, so the port layout, RID2SID count and tunnel reset + * registers are those of T8132. Two steps differ from the T8132 Linux + * sequence: the port must report RUN before the root-port config + * tunables are written, and the tunneled reset acknowledgment is the + * ACK_PEND bit being set. The block needs its port and its clocks + * disabled before sleep and the full enable sequence on wake, so + * hibernation stops the port and cold-starts it again. + */ static const struct hw_info t6040_pciec_hw = { .port_msiaddr = PORT_MSIADDR, .port_msiaddr_hi = PORT_MSIADDR + 4, @@ -418,6 +445,9 @@ static const struct hw_info t6040_pciec_hw = { .t8132_reset = true, .msimap_by_vector = true, .s2idle_hold_domain = true, + .tunnel_ready_before_rc = true, + .tunnel_ack_on_set = true, + .tunnel_hibernate = true, }; /* @@ -552,6 +582,8 @@ struct apple_pcie { bool neo_rc_down; /* Down already when the sleep began: a failed wake left it unprogrammed */ bool neo_rc_was_down; + /* Functions' saved state, kept from a restore's noirq phase to its link */ + struct list_head restore_fns; const struct hw_info *hw; unsigned long *bitmap; struct list_head ports; @@ -592,6 +624,28 @@ struct apple_pcie_port { unsigned long adopted_slots; /* adopt_link: RID2SID slots routed (slot == SID) */ struct gpio_desc *pwren; /* adopt_link: endpoint power, if Linux trained */ bool powered; /* cold train: pwren raised, port not parked yet */ + /* + * Conventional port: what the boot loader set on the port and its root port, which a + * sleep that powers the block down loses. + */ + struct apple_tunable *s2r_port_tunable; + struct apple_tunable *s2r_rc_tunable[3]; + u32 s2r_port_130; + u32 s2r_dbi[5]; + bool s2r_saved; + /* Conventional port: link up when the image was taken */ + bool restore_link_up; + /* ... and found down by the restore: the boot that resumed did not set it up */ + bool restore_cold; + /* The result of setting it up again (apple_pcie_conventional_up()) */ + int restore_ret; +}; + +/* A function's configuration, for its link back after a resume from disk. */ +struct apple_pcie_restore_fn { + struct list_head entry; + struct pci_dev *pdev; + struct pci_saved_state *state; }; static bool apple_radio_writer_closed(struct apple_pcie *pcie); @@ -1328,9 +1382,9 @@ static int apple_pcie_tunnel_cold_init(struct apple_pcie_port *port) /* * Without an m1n1 handoff, T600x/T602x ports must report RUN before * the root complex config space is written: writing the RC tunables - * first hard-resets a J416c. + * first hard-resets a J416c. T6040 needs the wait for RUN there too. */ - if (!pcie->kernel_init) { + if (!pcie->kernel_init || pcie->hw->tunnel_ready_before_rc) { ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, stat & PORT_STATUS_READY, 10, 250000, false, port, @@ -1339,9 +1393,9 @@ static int apple_pcie_tunnel_cold_init(struct apple_pcie_port *port) return dev_err_probe(pcie->dev, ret, "port %pOF cold init: RUN not set (status %#x)\n", port->np, stat); - } else { - apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); } + if (pcie->kernel_init) + apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); dev_info(pcie->dev, "port %pOF cold init: %sroot tunables\n", port->np, pcie->oe_fabric_base ? "oe-fabric and " : ""); @@ -1704,9 +1758,12 @@ static int apple_pcie_tunnel_stop(struct apple_pcie_port *port) err = ret; apple_pcie_port_rmw_set(port, PORT_TUNCTRL_PERST_ACK_REQ, apple_pcie_port_tunctrl(port)); - /* M1 reports an outstanding acknowledgment until ACK_PEND clears. */ + /* + * M1 reports an outstanding acknowledgment until ACK_PEND clears. + * T6040 reports the acknowledgment by setting it. + */ ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, - pcie->kernel_init ? + pcie->kernel_init && !pcie->hw->tunnel_ack_on_set ? !(stat & PORT_TUNSTAT_PERST_ACK_PEND) : stat & PORT_TUNSTAT_PERST_ACK_PEND, 1000, 1000000, false, port, apple_pcie_port_tunstat(port)); @@ -2355,29 +2412,27 @@ static int apple_pcie_t6050_port_train(struct apple_pcie_port *port) #define T6050_PHY_CTRL_LANES GENMASK(9, 8) #define T6050_PHY_CFG 0x4 /* - * Set before the PHY-IP tunables. A trace of macOS bringing the port up shows no - * access to this bit; meaning not known. + * Set before the PHY-IP tunables; meaning not known. */ #define T6050_PHY_CFG_BIT0 BIT(0) /* - * Set after the PHY-IP tunables. A trace of macOS shows it set too, after which - * T6050_PHY_STAT_BIT0 rises (the trace reads 0x10 seven times, then 0x11); meaning - * not known. + * Set after the PHY-IP tunables, after which T6050_PHY_STAT_BIT0 rises (the status + * reads 0x10 several times, then 0x11); meaning not known. */ #define T6050_PHY_CFG_BIT4 BIT(4) #define T6050_PHY_STAT 0x8 -/* Rises once T6050_PHY_CFG_BIT4 is set; polled by macOS and by m1n1; meaning not known */ +/* Rises once T6050_PHY_CFG_BIT4 is set; m1n1 polls it too; meaning not known */ #define T6050_PHY_STAT_BIT0 BIT(0) /* - * Set and held once the shared PHY is started in macOS's post-iBoot order (aura's + * Set and held once the shared PHY is started after iBoot (aura's * T6050 bring-up sets T6050_PHY_CFG_BIT4 and waits for it); meaning not known */ #define T6050_PHY_STAT_BIT4 BIT(4) #define T6050_PHYCMN_CLK 0x0 #define T6050_PHYCMN_CLK_MODE GENMASK(1, 0) /* - * Written before CORE_RC_CTL_RUN: a trace of macOS shows 0x140 written here, and m1n1 - * writes the same. Neither bit's meaning is known. + * Written before CORE_RC_CTL_RUN with 0x140, as m1n1 does. Neither bit's meaning is + * known. */ #define CORE_RC_T6050_PRE_RUN 0x54 #define CORE_RC_T6050_PRE_RUN_BIT6 BIT(6) @@ -2464,7 +2519,7 @@ static int apple_pcie_t6050_shared_phy_on(struct apple_pcie *pcie) T6050_PHY_CTRL_RESET_T8103)) == up) return 0; /* - * A boot loader that started the PHY in macOS's order (it trained another port) + * A boot loader that started the PHY already (it trained another port) * leaves status bit 4 set and the RC ready: the PHY runs, so it is not restarted * under that port's link. */ @@ -2604,7 +2659,7 @@ static int apple_pcie_t6050_cold_train(struct apple_pcie_port *port) /* * PME_Turn_Off, then wait for L2: the clock gate disabled, the message sent, and the * link reported in L2. Both waits are made even when the first fails (the N1 may - * clear the send bit and never reach L2, as under macOS); the first error is returned. + * clear the send bit and never reach L2); the first error is returned. */ static int apple_pcie_t6050_port_l2(struct apple_pcie_port *port) { @@ -2913,6 +2968,53 @@ static int apple_pcie_port_reset_endpoint(struct apple_pcie *pcie, stat & PORT_STATUS_READY, 100, 250000); } +/* Let the port gate its clocks again once the link is set up. */ +static void apple_pcie_port_gate_clocks(struct apple_pcie *pcie, + struct apple_pcie_port *port) +{ + if (pcie->hw->retain_clocks) + return; + if (pcie->hw->port_refclk) + rmw_clear(PORT_REFCLK_CGDIS, port->base + pcie->hw->port_refclk); + else if (port->phy) + rmw_set(PHY_LANE_CFG_REFCLKCGEN, port->phy + PHY_LANE_CFG); + + /* PCIe-C APPCLK state is part of the m1n1 handoff. */ + if (!pcie->hw->tunneled) + rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); +} + +/* + * The tunables the loader applied to a conventional port and its root port at boot, where it + * publishes them: written again after a sleep that reset the block. A missing property is + * fine, a malformed one fails the probe. + */ +static int apple_pcie_port_s2r_tunables(struct apple_pcie *pcie, struct apple_pcie_port *port, + struct resource *port_res) +{ + static const char * const rc_names[] = { + "apple,tunable-rc", "apple,tunable-rc-gen3", "apple,tunable-rc-gen4", + }; + struct resource rc = DEFINE_RES_MEM(0, SZ_4K); + struct apple_tunable *t; + int i; + + t = devm_apple_tunable_parse(pcie->dev, port->np, "apple,tunable-port", port_res); + if (IS_ERR(t) && PTR_ERR(t) != -ENOENT) + return dev_err_probe(pcie->dev, PTR_ERR(t), "port %pOF: apple,tunable-port\n", + port->np); + port->s2r_port_tunable = IS_ERR(t) ? NULL : t; + for (i = 0; i < ARRAY_SIZE(rc_names); i++) { + t = devm_apple_tunable_parse(pcie->dev, port->np, rc_names[i], &rc); + if (IS_ERR(t) && PTR_ERR(t) != -ENOENT) + return dev_err_probe(pcie->dev, PTR_ERR(t), "port %pOF: %s\n", + port->np, rc_names[i]); + port->s2r_rc_tunable[i] = IS_ERR(t) ? NULL : t; + } + + return 0; +} + static int apple_pcie_setup_port(struct apple_pcie *pcie, struct device_node *np) { @@ -2976,6 +3078,11 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (ret) goto err_teardown; } + if (!pcie->hw->tunneled && !pcie->hw->adopt_link && !pcie->managed_sids) { + ret = apple_pcie_port_s2r_tunables(pcie, port, res); + if (ret) + goto err_teardown; + } if (pcie->hw->tunneled) { port->tunable = devm_apple_tunable_parse(pcie->dev, np, "apple,tunable", res); @@ -3078,18 +3185,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } } - if (!pcie->hw->retain_clocks) { - if (pcie->hw->port_refclk) - rmw_clear(PORT_REFCLK_CGDIS, - port->base + pcie->hw->port_refclk); - else if (port->phy) - rmw_set(PHY_LANE_CFG_REFCLKCGEN, - port->phy + PHY_LANE_CFG); - - /* PCIe-C APPCLK state is part of the m1n1 handoff. */ - if (!pcie->hw->tunneled) - rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); - } + apple_pcie_port_gate_clocks(pcie, port); ret = apple_pcie_port_setup_irq(port); if (ret) @@ -3314,31 +3410,35 @@ static int apple_msi_init(struct apple_pcie *pcie) if (of_address_to_resource(to_of_node(fwnode), 0, &ecam)) goto invalid_slice; /* These ECAM apertures identify the ADT bridges of each SoC. */ - if (of_machine_is_compatible("apple,t6030")) { - aic_base = 1342; + if (pcie->hw == &t6040_pciec_hw) { + /* + * ADT msi-vector-offset 0x817. The internal port owns + * vectors 0..31, bridge pcicN 0x20 * (N + 1). + */ + aic_base = 2071; switch (ecam.start) { - case 0x730000000ULL: + case 0x1c50000000ULL: expected_base = 0x20; break; - case 0xb30000000ULL: + case 0x1c60000000ULL: expected_base = 0x40; break; - case 0xf30000000ULL: + case 0x1c70000000ULL: expected_base = 0x60; break; default: goto invalid_slice; } - } else if (of_machine_is_compatible("apple,t6040")) { - aic_base = 2071; + } else if (of_machine_is_compatible("apple,t6030")) { + aic_base = 1342; switch (ecam.start) { - case 0x1c50000000ULL: + case 0x730000000ULL: expected_base = 0x20; break; - case 0x1c60000000ULL: + case 0xb30000000ULL: expected_base = 0x40; break; - case 0x1c70000000ULL: + case 0xf30000000ULL: expected_base = 0x60; break; default: @@ -4518,7 +4618,7 @@ static int apple_pcie_neo_phy_clocks(void __iomem *phy) } /* - * The part of the block that ANS shares, from power-off in macOS's order, measured on + * The part of the block that ANS shares, from power-off in the order it needs, measured on * one sleep. The sleep driver does it at the wake when ANS is bound, and it can't be * redone under a running PHY. */ @@ -4599,7 +4699,7 @@ static int apple_pcie_neo_cold_init(struct apple_pcie *pcie, struct apple_pcie_p apple_tunable_apply(port->base, cold->port_tunable); rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); - /* PERST# off before the PHY's clocks, as macOS has it coming from power-off. */ + /* PERST# off before the PHY's clocks, the order the block needs coming from power-off. */ rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, stat & PORT_STATUS_READY, 100, NEO_RC_POLL_US); @@ -4988,6 +5088,7 @@ static int apple_pcie_probe(struct platform_device *pdev) mutex_init(&pcie->lock); INIT_LIST_HEAD(&pcie->ports); + INIT_LIST_HEAD(&pcie->restore_fns); /* Resolve PM dependencies before publishing the PCI hierarchy. */ if (pcie->hw->tunneled) { @@ -5786,6 +5887,335 @@ static void apple_pcie_stop_for_sleep(struct device *dev) static void apple_pcie_neo_save_functions(struct apple_pcie *pcie); +/* + * A conventional host (the internal ports with their radio and SD reader) keeps running + * through freeze and thaw. Only a kernel restored from an image finds it changed: the boot + * that resumed it went through the boot loader, and when its kernel did not set the ports up + * (a boot kernel without the GPIO drivers that probing the host takes), the endpoints sit + * unpowered or in reset behind a link that is down. The restore then does what probe did. + * A real suspend-to-RAM takes the host's power as a resume from disk does, so its ports + * come back the same way. + */ +static bool apple_pcie_conventional_host(struct apple_pcie *pcie) +{ + return pcie->hw->conventional_restore && !pcie->managed_sids; +} + +/* + * Root port words the loader limits at boot (one lane at the endpoint's speed) and a reset + * puts back to the core's own: link capabilities, the target speed, link mode and lane count. + */ +static const u16 apple_pcie_s2r_dbi[] = { + NEO_DBI_CAP_LNKCAP, NEO_DBI_CAP_LNKCAP2, NEO_DBI_CAP_LNKCTL2, + NEO_DBI_PORT_LINK_CONTROL, NEO_DBI_WIDTH_SPEED_CONTROL, +}; + +/* Bounded like probe's wait, polled: the link interrupts stay masked meanwhile. */ +static int apple_pcie_restore_train(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + int attempt, attempts = port->perst ? 1 + LINK_RESET_RETRIES : 1; + u32 stat; + int ret = -ETIMEDOUT; + + for (attempt = 0; attempt < attempts; attempt++) { + if (attempt) { + dev_info(pcie->dev, "%pOF link down after %dms (LTSSM %#x), resetting the endpoint\n", + port->np, link_up_timeout, apple_pcie_port_ltssm(pcie, port)); + ret = apple_pcie_port_reset_endpoint(pcie, port); + if (ret) + return ret; + } + apple_pcie_port_writel(port, PORT_LTSSMCTL_START, PORT_LTSSMCTL); + ret = readl_relaxed_poll_timeout(port->base + PORT_LINKSTS, stat, + stat & PORT_LINKSTS_UP, 100, + link_up_timeout * 1000); + if (!ret) + break; + } + return ret; +} + +/* This kernel's RID-to-SID table, every other entry cleared as probe clears them. */ +static void apple_pcie_restore_rid2sid(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + u32 val, got, want = 0, bad = 0, first = 0; + int i, first_idx = -1; + + mutex_lock(&pcie->lock); + for (i = 0; i < port->sid_map_sz; i++) { + val = test_bit(i, port->sid_map) ? port->saved_rid2sid[i] : 0; + got = apple_pcie_rid2sid_write(port, i, val); + if (got == val) + continue; + if (!bad++) { + first_idx = i; + want = val; + first = got; + } + } + mutex_unlock(&pcie->lock); + if (bad) + dev_warn(pcie->dev, + "%pOF: %u RID-to-SID word(s) did not read back as written after the restore (first: entry %d wrote %#x, read %#x)\n", + port->np, bad, first_idx, want, first); +} + +/* + * What both ways back have in common once the port runs with its root port set: clocks gated + * as at probe, all port interrupts masked and cleared with the MSI routing as at probe, the + * RID-to-SID table, the link trained, and the interrupt mask the kernel had. + */ +static int apple_pcie_restore_port_finish(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + int ret; + + apple_pcie_port_gate_clocks(pcie, port); + apple_pcie_tunnel_restore_irq_hw(port); + apple_pcie_restore_rid2sid(pcie, port); + ret = apple_pcie_restore_train(pcie, port); + if (ret) + return ret; + apple_pcie_port_writel(port, BIT(PORT_INT_LINK_UP), PORT_INTSTAT); + apple_pcie_port_writel(port, port->saved_intmask, PORT_INTMSK); + return 0; +} + +static void apple_pcie_s2r_save_port(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + void __iomem *dbi = pcie->cfg->win + ((resource_size_t)port->idx << 15); + int i; + + BUILD_BUG_ON(ARRAY_SIZE(apple_pcie_s2r_dbi) != ARRAY_SIZE(port->s2r_dbi)); + /* + * Read at every sleep, not once at probe: after the loader, these words are written again + * by the cold initialization of a port (link capabilities, mode, width, target speed), + * by the PCI core through the root port's config space (Link Control 2) and by a tunnel's + * setup (0x130), so the restore has to reproduce the values of the moment. + * The loader's port setup writes 0x130; the port tunable has its own word. + */ + port->s2r_port_130 = apple_pcie_port_readl(port, 0x130); + for (i = 0; i < ARRAY_SIZE(apple_pcie_s2r_dbi); i++) + port->s2r_dbi[i] = readl_relaxed(dbi + apple_pcie_s2r_dbi[i]); + port->s2r_saved = true; +} + +/* The root port part goes in once the port runs again, before the link starts. */ +static void apple_pcie_s2r_restore_rc(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + void __iomem *dbi = pcie->cfg->win + ((resource_size_t)port->idx << 15); + int i; + + rmw_set(NEO_DBI_RO_WR_EN, dbi + NEO_DBI_RO_WR); + for (i = 0; i < ARRAY_SIZE(port->s2r_rc_tunable); i++) + apple_tunable_apply(dbi, port->s2r_rc_tunable[i]); + for (i = 0; i < ARRAY_SIZE(apple_pcie_s2r_dbi); i++) { + /* Link Control 2 shares its word with a status register. */ + if (apple_pcie_s2r_dbi[i] == NEO_DBI_CAP_LNKCTL2) + writew_relaxed(port->s2r_dbi[i], dbi + apple_pcie_s2r_dbi[i]); + else + writel_relaxed(port->s2r_dbi[i], dbi + apple_pcie_s2r_dbi[i]); + } + /* The link goes to the target speed once it is up, as after the loader's setup. */ + rmw_set(NEO_DBI_SPEED_CHANGE, dbi + NEO_DBI_WIDTH_SPEED_CONTROL); + rmw_clear(NEO_DBI_RO_WR_EN, dbi + NEO_DBI_RO_WR); +} + +/* + * A real S2R took the port block's power: the sleep's wake side clocked the port again and + * the pin restore released PERST#, with the link down and every word of the port and of its + * root port at its reset value. Do what the loader and probe did between them, here, so that + * the PCI core's noirq pass finds the functions: PERST# asserted again, the port's and the + * root port's settings back, PERST# released, the link trained. The endpoint's power line + * is not touched: it is the SMC's, whose session is back only in the normal phase, and it + * reads on after the wake. + * + * In two steps, so that the ports share the two waits instead of each taking its own: up to + * the release of PERST#, and from the port's ready bit on. + */ +static int apple_pcie_s2r_port_release_noirq(struct apple_pcie *pcie, + struct apple_pcie_port *port) +{ + int ret; + + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + gpiod_set_value(port->perst_gpio, 1); + if (!pcie->hw->retain_clocks) { + ret = apple_pcie_setup_refclk(pcie, port); + if (ret < 0) + return ret; + } + apple_pcie_port_writel(port, port->s2r_port_130, 0x130); + apple_tunable_apply(port->base, port->s2r_port_tunable); + /* Tperst-clk (PCIe CEM r5.0, 2.9.2) */ + usleep_range(100, 200); + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + gpiod_set_value(port->perst_gpio, 0); + return 0; +} + +static int apple_pcie_s2r_port_train_noirq(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + u32 stat; + int ret; + + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, + stat & PORT_STATUS_READY, 100, 250000); + if (ret) + return ret; + apple_pcie_s2r_restore_rc(pcie, port); + return apple_pcie_restore_port_finish(pcie, port); +} + +static void apple_pcie_s2r_port_failed(struct apple_pcie *pcie, struct apple_pcie_port *port, + int ret) +{ + dev_warn(pcie->dev, "S2R: %pOF link not back in the noirq phase (%d, LTSSM %#x)\n", + port->np, ret, apple_pcie_port_ltssm(pcie, port)); +} + +/* A port that does not come up here is left to the normal phase, as before. */ +static void apple_pcie_s2r_up_noirq(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + unsigned long released = 0, trained = 0; + bool fast = false; + int ret; + + list_for_each_entry(port, &pcie->ports, entry) { + if (!port->restore_link_up || !port->s2r_saved || !port->perst_gpio || + gpiod_cansleep(port->perst_gpio) || port->idx >= BITS_PER_LONG || + (apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP)) + continue; + ret = apple_pcie_s2r_port_release_noirq(pcie, port); + if (ret) + apple_pcie_s2r_port_failed(pcie, port, ret); + else + released |= BIT(port->idx); + } + if (!released) + return; + /* 100 ms after PERST# (PCIe r5.0, 6.6.1), once for all of them. */ + msleep(100); + list_for_each_entry(port, &pcie->ports, entry) { + if (port->idx >= BITS_PER_LONG || !(released & BIT(port->idx))) + continue; + ret = apple_pcie_s2r_port_train_noirq(pcie, port); + if (ret) + apple_pcie_s2r_port_failed(pcie, port, ret); + else + trained |= BIT(port->idx); + } + /* + * A link above 5 GT/s takes no config request before 100 ms after its training + * (PCIe r5.0, 6.6.1). Up to 5 GT/s the time since PERST# is what counts, and that + * has passed. + */ + list_for_each_entry(port, &pcie->ports, entry) { + void __iomem *dbi = pcie->cfg->win + ((resource_size_t)port->idx << 15); + + if (port->idx >= BITS_PER_LONG || !(trained & BIT(port->idx))) + continue; + /* The target, not the current speed: the change of speed may still run. */ + if ((readw_relaxed(dbi + NEO_DBI_CAP_LNKCTL2) & PCI_EXP_LNKCTL2_TLS) > + PCI_EXP_LNKCTL2_TLS_5_0GT) + fast = true; + } + if (fast) + msleep(100); + list_for_each_entry(port, &pcie->ports, entry) { + if (port->idx >= BITS_PER_LONG || !(trained & BIT(port->idx))) + continue; + dev_info(pcie->dev, "S2R: %pOF link back with the loader's port settings\n", + port->np); + } +} + +/* What only the port registers hold, for a restore that finds them reset. */ +static void apple_pcie_restore_save(struct apple_pcie *pcie) +{ + struct apple_pcie_port *port; + int i; + + mutex_lock(&pcie->lock); + list_for_each_entry(port, &pcie->ports, entry) { + apple_pcie_s2r_save_port(pcie, port); + port->restore_link_up = apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP; + port->restore_cold = false; + port->saved_intmask = apple_pcie_port_readl(port, PORT_INTMSK); + for_each_set_bit(i, port->sid_map, port->sid_map_sz) + port->saved_rid2sid[i] = + apple_pcie_port_readl(port, port_rid2sid_offset(port, i)); + } + mutex_unlock(&pcie->lock); +} + +static void apple_pcie_restore_free_fns(struct apple_pcie *pcie) +{ + struct apple_pcie_restore_fn *fn, *tmp; + + list_for_each_entry_safe(fn, tmp, &pcie->restore_fns, entry) { + list_del(&fn->entry); + kfree(fn->state); + pci_dev_put(fn->pdev); + kfree(fn); + } +} + +/* + * The PCI core's noirq restore spends the functions' saved state on a link that is down + * ("device inaccessible"). A copy taken first goes back once the link is: the core's normal + * restore then restores it (pci_pm_restore() restores a function whose state is saved). + */ +static int apple_pcie_restore_keep_fn(struct pci_dev *pdev, void *data) +{ + struct apple_pcie *pcie = data; + struct apple_pcie_port *port; + struct apple_pcie_restore_fn *fn; + + if (pci_is_root_bus(pdev->bus) || !pdev->state_saved) + return 0; + port = apple_pcie_get_port(pdev); + if (!port || !port->restore_cold) + return 0; + fn = kzalloc_obj(*fn); + if (!fn) + return 0; + fn->state = pci_store_saved_state(pdev); + if (!fn->state) { + kfree(fn); + return 0; + } + fn->pdev = pci_dev_get(pdev); + list_add_tail(&fn->entry, &pcie->restore_fns); + return 0; +} + +/* + * Before the PCI core restores the functions. Reading the link state is what probe does on + * every cold boot; nothing is written to a port in this phase. Its PWREN is an SMC GPIO, and + * the SMC session is back only in the normal phase. + */ +static void apple_pcie_restore_check_noirq(struct apple_pcie *pcie, const char *what) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + struct apple_pcie_port *port; + bool cold = false; + + apple_pcie_restore_free_fns(pcie); + list_for_each_entry(port, &pcie->ports, entry) { + if (!port->restore_link_up) + continue; + if (apple_pcie_port_readl(port, PORT_LINKSTS) & PORT_LINKSTS_UP) + continue; + port->restore_cold = true; + cold = true; + dev_info(pcie->dev, "%s: %pOF lost its link, setting it up again\n", + what, port->np); + } + if (cold && bridge->bus) + pci_walk_bus(bridge->bus, apple_pcie_restore_keep_fn, pcie); +} + /* * The J716c radio drivers save PCI state without entering D3. The PCI core * keeps the root port in D0 as well. Its host and DART have no switchable @@ -5892,6 +6322,8 @@ static int apple_pcie_suspend_noirq(struct device *dev) if (!pcie->hw->tunneled) { if (pm_suspend_no_platform() && pcie->num_pds) return apple_pcie_hold_domains(pcie); + if (apple_pcie_conventional_host(pcie) && pm_suspend_via_firmware()) + apple_pcie_restore_save(pcie); return 0; } pcie->reset_on_resume = false; @@ -5962,7 +6394,7 @@ static int apple_pcie_neo_link_up(struct apple_pcie *pcie, struct apple_pcie_por if (ret) return ret; released = ktime_get(); - /* macOS starts the training right after PERST#. */ + /* The training starts right after PERST#. */ ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, stat & PORT_STATUS_READY, 100, 250000); if (ret) @@ -6139,8 +6571,8 @@ static void apple_pcie_neo_radio_work(struct work_struct *work) } /* - * The rail is the one thing that brings back a chip whose link won't train: macOS's last resort - * for a dead chip too. A cold chip then has its firmware loaded by the probe that follows. + * The rail is the one thing that brings back a chip whose link won't train, a dead chip + * too. A cold chip then has its firmware loaded by the probe that follows. */ static int apple_pcie_neo_rail_cycle(struct apple_pcie *pcie, struct apple_pcie_port *port, struct pci_dev *root) @@ -6262,8 +6694,8 @@ static void apple_pcie_neo_recover_work(struct work_struct *work) } /* - * Before the PCI core restores the functions, which need the link. macOS's host bridge - * holds them back the same way until its radio port is enabled. + * Before the PCI core restores the functions, which need the link. They are held back + * until the radio port is enabled. */ static int apple_pcie_neo_resume_noirq(struct apple_pcie *pcie) { @@ -6382,8 +6814,13 @@ static int apple_pcie_resume_noirq(struct device *dev) } return ret; } - if (!pcie->hw->tunneled) + if (!pcie->hw->tunneled) { + if (apple_pcie_conventional_host(pcie) && pm_resume_via_firmware()) { + apple_pcie_s2r_up_noirq(pcie); + apple_pcie_restore_check_noirq(pcie, "S2R"); + } return 0; + } if (pcie->resume_failed) return -EIO; /* @@ -6450,6 +6887,25 @@ static int apple_pcie_resume_noirq(struct device *dev) return ret; } +/* Probe's bring-up of a conventional port, with the state this kernel had programmed. */ +static int apple_pcie_restore_port_up(struct apple_pcie *pcie, struct apple_pcie_port *port) +{ + int ret; + + /* A port with auxiliary resets does not keep its GPIOs for a second bring-up. */ + if (!port->perst_gpio) + return -EOPNOTSUPP; + ret = apple_pcie_setup_link(pcie, port, port->np); + if (ret) + return ret; + ret = apple_pcie_restore_port_finish(pcie, port); + if (ret) + return ret; + /* 100 ms after the link before configuration requests (PCIe r5.0, 6.6.1). */ + msleep(100); + return 0; +} + static int apple_pcie_prepare(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); @@ -6468,12 +6924,151 @@ static int apple_pcie_prepare(struct device *dev) return apple_radio_prepare(dev); } +static void apple_pcie_restore_port_up_async(void *data, async_cookie_t cookie) +{ + struct apple_pcie_port *port = data; + + port->restore_ret = apple_pcie_restore_port_up(port->pcie, port); +} + +/* + * The normal phase, after the SMC's own restore (the PWREN GPIOs are its) and before the PCI + * core's normal restore of the functions, which waits for this host as their ancestor. + * + * Each port's bring-up is mostly the waits of the PCIe timing (power to PERST#, PERST# to + * the link, the link to the first configuration request) and touches only that port's own + * registers, GPIOs and RID-to-SID table (under the host's lock): the ports are brought up + * side by side and their waits overlap. + */ +static void apple_pcie_conventional_up(struct apple_pcie *pcie, const char *what) +{ + ASYNC_DOMAIN_EXCLUSIVE(ports_up); + struct device *dev = pcie->dev; + struct apple_pcie_port *port; + struct apple_pcie_restore_fn *fn; + unsigned int back = 0; + int ret; + + list_for_each_entry(port, &pcie->ports, entry) { + if (!port->restore_link_up) + continue; + if (!port->restore_cold) { + /* + * Up: set up by the boot that resumed, or never reset. The functions + * this kernel enabled keep their mappings only if the table is its own. + */ + apple_pcie_restore_rid2sid(pcie, port); + continue; + } + async_schedule_domain(apple_pcie_restore_port_up_async, port, &ports_up); + } + async_synchronize_full_domain(&ports_up); + + list_for_each_entry(port, &pcie->ports, entry) { + if (!port->restore_link_up || !port->restore_cold) + continue; + ret = port->restore_ret; + if (ret) { + dev_err(dev, "%s: %pOF link not back (%d), its functions stay unreachable\n", + what, port->np, ret); + port->restore_cold = false; + continue; + } + dev_info(dev, "%s: %pOF link back, PWREN and PERST# cycled as at probe\n", + what, port->np); + back++; + } + list_for_each_entry(fn, &pcie->restore_fns, entry) { + port = apple_pcie_get_port(fn->pdev); + if (!port || !port->restore_cold) + continue; + if (pci_load_saved_state(fn->pdev, fn->state)) + pci_warn(fn->pdev, "%s: saved configuration not taken back\n", what); + } + apple_pcie_restore_free_fns(pcie); + list_for_each_entry(port, &pcie->ports, entry) + port->restore_cold = false; + if (back) + dev_dbg(dev, "%s: %u port(s) set up again\n", what, back); +} + +static int apple_pcie_restore(struct device *dev) +{ + struct apple_pcie *pcie = apple_pcie_lookup(dev); + + if (!apple_pcie_conventional_host(pcie)) + return 0; + apple_pcie_conventional_up(pcie, "resume from disk"); + return 0; +} + +/* The normal phase of a real S2R's resume, ordered as the restore above. */ +static int apple_pcie_resume(struct device *dev) +{ + struct apple_pcie *pcie = apple_pcie_lookup(dev); + + if (!apple_pcie_conventional_host(pcie) || !pm_resume_via_firmware()) + return 0; + apple_pcie_conventional_up(pcie, "S2R"); + return 0; +} + +/* + * Hibernation, for the controllers that opt in: freeze and poweroff stop the + * tunnel as a system sleep that keeps nothing does, thaw and restore start it + * again. After a restore the controller comes back from a power cycle, so the + * start replays the whole cold sequence. A conventional host only saves what a + * restore over a cold boot needs, see apple_pcie_conventional_host(). + */ +static int apple_pcie_hibernate_noirq(struct device *dev) +{ + struct apple_pcie *pcie = apple_pcie_lookup(dev); + + if (!pcie->hw->tunnel_hibernate) { + if (apple_pcie_conventional_host(pcie)) + apple_pcie_restore_save(pcie); + return 0; + } + pcie->reset_on_resume = false; + pcie->link_kept = false; + apple_pcie_stop_for_sleep(dev); + return 0; +} + +static int apple_pcie_wake_noirq(struct device *dev) +{ + struct apple_pcie *pcie = apple_pcie_lookup(dev); + + if (!pcie->hw->tunnel_hibernate) + return 0; + return apple_pcie_resume_noirq(dev); +} + +/* Thaw keeps every port as it was: only a restored image looks. */ +static int apple_pcie_restore_noirq(struct device *dev) +{ + struct apple_pcie *pcie = apple_pcie_lookup(dev); + + if (!pcie->hw->tunnel_hibernate) { + if (apple_pcie_conventional_host(pcie)) + apple_pcie_restore_check_noirq(pcie, "resume from disk"); + return 0; + } + return apple_pcie_resume_noirq(dev); +} + static const struct dev_pm_ops apple_pcie_pm_ops = { .prepare = apple_pcie_prepare, .complete = apple_radio_complete, .suspend_noirq = apple_pcie_suspend_noirq, .resume_noirq = apple_pcie_resume_noirq, .resume_early = apple_pcie_resume_early, + .resume = apple_pcie_resume, + .freeze_noirq = apple_pcie_hibernate_noirq, + .thaw_noirq = apple_pcie_wake_noirq, + .poweroff_noirq = apple_pcie_hibernate_noirq, + .restore_noirq = apple_pcie_restore_noirq, + .restore = apple_pcie_restore, }; static const struct of_device_id apple_pcie_of_match[] = { diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index 381c830e63978..196ded718c165 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -60,9 +60,10 @@ config PHY_APPLE_T6040_KUNIT_TEST default KUNIT_ALL_TESTS help Test the bounded hard-recovery state machine for a missed Power Down - acknowledgment, a failed firmware tune and retry exhaustion. The tests - access no hardware and run when the PHY driver is loaded. They are only - useful for kernel developers. + acknowledgment, a failed firmware tune and retry exhaustion, and which + port states the PHY takes through a deep sleep with the USB host kept. + The tests access no hardware and run when the PHY driver is loaded. + They are only useful for kernel developers. config PHY_APPLE_DPTX tristate "Apple DPTX PHY" diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 414ce695f37a2..a8beddfe9b8fa 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -4584,11 +4584,12 @@ static int atcphy_rpm_usb2_set_mode(struct phy *phy, enum phy_mode mode, int sub return ret; } -static const struct phy_ops apple_atc_usb2_phy_ops = { +const struct phy_ops apple_atc_usb2_phy_ops = { .owner = THIS_MODULE, .init = atcphy_rpm_usb2_init, .set_mode = atcphy_rpm_usb2_set_mode, }; +EXPORT_SYMBOL_GPL(apple_atc_usb2_phy_ops); static int atcphy_usb3_power_off(struct phy *phy) { diff --git a/drivers/phy/apple/t6040-recovery-test.c b/drivers/phy/apple/t6040-recovery-test.c index 552afba86f117..76b50f35cba2b 100644 --- a/drivers/phy/apple/t6040-recovery-test.c +++ b/drivers/phy/apple/t6040-recovery-test.c @@ -1,5 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -/* Included by t6040-usb2.c to test its recovery state machine. */ +/* + * Included by t6040-usb2.c to test its recovery state machine and which ports + * it takes through a sleep with the host kept. + */ #include @@ -35,10 +38,79 @@ static void t6040_recovery_retry_exhaustion_test(struct kunit *test) KUNIT_EXPECT_EQ(test, state.retries, (u8)T6040_RECOVERY_RETRIES); } +/* A sleep that powers the SoC down is the power cycle an unclean stop asks for. */ +static void t6040_recovery_power_lost_test(struct kunit *test) +{ + struct t6040_recovery_state state = {}; + + t6040_recovery_power_down(&state, true, false); + KUNIT_EXPECT_TRUE(test, state.cycle_pending); + t6040_recovery_power_lost(&state); + KUNIT_EXPECT_FALSE(test, state.cycle_pending); + KUNIT_EXPECT_FALSE(test, t6040_recovery_start(&state)); +} + +static void t6040_recovery_held_reference_test(struct kunit *test) +{ + unsigned int attempt; + unsigned int retries = t6040_recovery_pm_retries(false); + + KUNIT_EXPECT_EQ(test, retries, (unsigned int)T6040_RECOVERY_PM_RETRIES); + for (attempt = 0; attempt < retries; attempt++) + KUNIT_EXPECT_EQ(test, + t6040_recovery_pm_action(1, 0, attempt, retries), + T6040_RECOVERY_PM_RETRY); + KUNIT_EXPECT_EQ(test, + t6040_recovery_pm_action(1, 0, attempt, retries), + T6040_RECOVERY_PM_FALLBACK); + KUNIT_EXPECT_EQ(test, + t6040_recovery_pm_action(0, 1, attempt, retries), + T6040_RECOVERY_PM_FALLBACK); + KUNIT_EXPECT_EQ(test, + t6040_recovery_pm_action(0, 0, attempt, retries), + T6040_RECOVERY_PM_SUSPEND); +} + +/* + * A PHY that is still in use holds the mailbox itself: the recovery doesn't + * wait for that reference, and suspends the mailbox at once if nothing holds + * it after all. + */ +static void t6040_recovery_own_reference_test(struct kunit *test) +{ + unsigned int retries = t6040_recovery_pm_retries(true); + + KUNIT_EXPECT_EQ(test, retries, 0U); + KUNIT_EXPECT_EQ(test, t6040_recovery_pm_action(1, 0, 0, retries), + T6040_RECOVERY_PM_FALLBACK); + KUNIT_EXPECT_EQ(test, t6040_recovery_pm_action(0, 1, 0, retries), + T6040_RECOVERY_PM_FALLBACK); + KUNIT_EXPECT_EQ(test, t6040_recovery_pm_action(0, 0, 0, retries), + T6040_RECOVERY_PM_SUSPEND); +} + +/* + * The coprocessor's power domain stays on while its firmware runs, whoever + * asks for the power-off: a suspend-to-idle with a host running does. A + * stopped coprocessor lets it go, and the other notifications pass. + */ +static void t6040_pd_stays_on_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, t6040_pd_stays_on(GENPD_NOTIFY_PRE_OFF, true)); + KUNIT_EXPECT_FALSE(test, t6040_pd_stays_on(GENPD_NOTIFY_PRE_OFF, false)); + KUNIT_EXPECT_FALSE(test, t6040_pd_stays_on(GENPD_NOTIFY_OFF, true)); + KUNIT_EXPECT_FALSE(test, t6040_pd_stays_on(GENPD_NOTIFY_PRE_ON, true)); + KUNIT_EXPECT_FALSE(test, t6040_pd_stays_on(GENPD_NOTIFY_ON, true)); +} + static struct kunit_case t6040_recovery_test_cases[] = { KUNIT_CASE(t6040_recovery_missed_ack_test), KUNIT_CASE(t6040_recovery_failed_tune_test), KUNIT_CASE(t6040_recovery_retry_exhaustion_test), + KUNIT_CASE(t6040_recovery_power_lost_test), + KUNIT_CASE(t6040_recovery_held_reference_test), + KUNIT_CASE(t6040_recovery_own_reference_test), + KUNIT_CASE(t6040_pd_stays_on_test), {} }; @@ -47,4 +119,299 @@ static struct kunit_suite t6040_recovery_test_suite = { .test_cases = t6040_recovery_test_cases, }; -kunit_test_suite(t6040_recovery_test_suite); +/* A Type-C port's PHY with a USB2 host running on it, as dwc3-apple's start leaves it. */ +static struct t6040_usb2_phy *t6040_sleep_test_phy(struct kunit *test) +{ + struct t6040_usb2_phy *tphy = kunit_kzalloc(test, sizeof(*tphy), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, tphy); + /* Never dereferenced: only that the PHY has a pipehandler counts. */ + tphy->pipehandler = (void __iomem *)tphy; + tphy->mode = PHY_MODE_USB_HOST; + tphy->powered = true; + tphy->dwc3_running = true; + return tphy; +} + +/* USB2 alone, and USB3 with the PIPE on the PHY, are the two states that sleep in place. */ +static void t6040_sleep_usb_test(struct kunit *test) +{ + struct t6040_usb2_phy *tphy = t6040_sleep_test_phy(test); + + KUNIT_EXPECT_TRUE(test, t6040_can_sleep_in_place(tphy)); + tphy->usb3_on = true; + tphy->usb3_pipe = true; + KUNIT_EXPECT_TRUE(test, t6040_can_sleep_in_place(tphy)); + tphy->usb3_up = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->pipe_switched = true; + KUNIT_EXPECT_TRUE(test, t6040_can_sleep_in_place(tphy)); +} + +/* No host, no controller out of reset, or the fixed DisplayPort PHY: nothing to keep. */ +static void t6040_sleep_no_host_test(struct kunit *test) +{ + struct t6040_usb2_phy *tphy = t6040_sleep_test_phy(test); + + tphy->powered = false; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->powered = true; + tphy->dwc3_running = false; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->dwc3_running = true; + tphy->pipehandler = NULL; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); +} + +/* + * DisplayPort's four lanes (pin assignments C and E), a DisplayPort link and + * USB4/Thunderbolt stop the host. Pin assignment D without a link does not: + * USB2 alone or with USB3, it is a USB port until the DP phy powers on. + */ +static void t6040_sleep_other_modes_test(struct kunit *test) +{ + struct t6040_usb2_phy *tphy = t6040_sleep_test_phy(test); + + tphy->dp_alt = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->dp_two_lanes = true; + KUNIT_EXPECT_TRUE(test, t6040_can_sleep_in_place(tphy)); + tphy->usb3_up = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->pipe_switched = true; + KUNIT_EXPECT_TRUE(test, t6040_can_sleep_in_place(tphy)); + tphy->dp_up = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->dp_alt = false; + tphy->dp_two_lanes = false; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->dp_up = false; + KUNIT_EXPECT_TRUE(test, t6040_can_sleep_in_place(tphy)); + + tphy->usb3_up = false; + tphy->pipe_switched = false; + tphy->usb3_lent = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->cio_up = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->usb3_lent = false; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->cio_up = false; + tphy->pipe_tunneled = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->pipe_tunneled = false; + tphy->router_running = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); +} + +/* A PHY that failed, is down for a sleep already, or didn't come back from one. */ +static void t6040_sleep_failed_test(struct kunit *test) +{ + struct t6040_usb2_phy *tphy = t6040_sleep_test_phy(test); + + tphy->shutdown_failed = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->shutdown_failed = false; + tphy->sleep_no_keep = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->sleep_no_keep = false; + tphy->sleep_down = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->sleep_down = false; + tphy->sleep_reset = true; + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + tphy->sleep_reset = false; + KUNIT_EXPECT_TRUE(test, t6040_can_sleep_in_place(tphy)); +} + +/* The port can't sleep in place, and the log names the conditions that fail as @names. */ +static void t6040_expect_miss(struct kunit *test, const struct t6040_usb2_phy *tphy, + const char *names) +{ + char why[T6040_SLEEP_MISSES_LEN]; + + t6040_sleep_in_place_misses(tphy, why, sizeof(why)); + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + KUNIT_EXPECT_STREQ(test, why, names); +} + +/* + * Each of the conditions of t6040_can_sleep_in_place() has its name in the + * log, all of them fit into the buffer, and a port that sleeps in place has + * none. + */ +static void t6040_sleep_misses_test(struct kunit *test) +{ + struct t6040_usb2_phy *tphy = t6040_sleep_test_phy(test); + char why[T6040_SLEEP_MISSES_LEN]; + + t6040_sleep_in_place_misses(tphy, why, sizeof(why)); + KUNIT_EXPECT_TRUE(test, t6040_can_sleep_in_place(tphy)); + KUNIT_EXPECT_STREQ(test, why, ""); + + tphy->pipehandler = NULL; + t6040_expect_miss(test, tphy, " no_pipehandler"); + tphy->pipehandler = (void __iomem *)tphy; + + tphy->powered = false; + t6040_expect_miss(test, tphy, " usb2_off"); + tphy->powered = true; + + tphy->dwc3_running = false; + t6040_expect_miss(test, tphy, " dwc3_in_reset"); + tphy->dwc3_running = true; + + tphy->mode = PHY_MODE_INVALID; + t6040_expect_miss(test, tphy, " mode"); + tphy->mode = PHY_MODE_USB_HOST; + + tphy->shutdown_failed = true; + t6040_expect_miss(test, tphy, " shutdown_failed"); + tphy->shutdown_failed = false; + + tphy->sleep_no_keep = true; + t6040_expect_miss(test, tphy, " sleep_no_keep"); + tphy->sleep_no_keep = false; + + tphy->sleep_down = true; + t6040_expect_miss(test, tphy, " sleep_down"); + tphy->sleep_down = false; + + tphy->sleep_reset = true; + t6040_expect_miss(test, tphy, " sleep_reset"); + tphy->sleep_reset = false; + + tphy->cio_up = true; + t6040_expect_miss(test, tphy, " cio_up"); + tphy->cio_up = false; + + tphy->pipe_tunneled = true; + t6040_expect_miss(test, tphy, " pipe_tunneled"); + tphy->pipe_tunneled = false; + + tphy->router_running = true; + t6040_expect_miss(test, tphy, " router_running"); + tphy->router_running = false; + + tphy->dp_up = true; + t6040_expect_miss(test, tphy, " dp_up"); + tphy->dp_up = false; + + tphy->dp_alt = true; + t6040_expect_miss(test, tphy, " dp_four_lanes"); + tphy->dp_alt = false; + + tphy->usb3_lent = true; + t6040_expect_miss(test, tphy, " usb3_lent"); + tphy->usb3_lent = false; + + tphy->usb3_up = true; + t6040_expect_miss(test, tphy, " pipe_state"); + tphy->usb3_up = false; + + tphy->pipe_switched = true; + t6040_expect_miss(test, tphy, " pipe_state"); + tphy->pipe_switched = false; + + /* Pin assignment D without a link fails nothing. */ + tphy->dp_alt = true; + tphy->dp_two_lanes = true; + t6040_sleep_in_place_misses(tphy, why, sizeof(why)); + KUNIT_EXPECT_TRUE(test, t6040_can_sleep_in_place(tphy)); + KUNIT_EXPECT_STREQ(test, why, ""); + tphy->dp_two_lanes = false; + + /* A display on all four lanes, with USB3's lanes lent to it. */ + tphy->dp_up = true; + tphy->usb3_lent = true; + t6040_expect_miss(test, tphy, " dp_up dp_four_lanes usb3_lent"); + + /* Everything at once. */ + tphy->pipehandler = NULL; + tphy->powered = false; + tphy->dwc3_running = false; + tphy->mode = PHY_MODE_INVALID; + tphy->shutdown_failed = true; + tphy->sleep_no_keep = true; + tphy->sleep_down = true; + tphy->sleep_reset = true; + tphy->cio_up = true; + tphy->pipe_tunneled = true; + tphy->router_running = true; + tphy->usb3_up = true; + t6040_expect_miss(test, tphy, + " no_pipehandler usb2_off dwc3_in_reset mode shutdown_failed sleep_no_keep sleep_down sleep_reset cio_up pipe_tunneled router_running dp_up dp_four_lanes usb3_lent pipe_state"); +} + +/* + * Only the controller's reset and USB2 can fail the keeping of a host, and + * only when the host was lost: not a sleep refused for a USB change, not what + * a wake that released the reset reports, and no host that was kept. + */ +static void t6040_sleep_keep_failed_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, t6040_sleep_keep_failed(false, -ENODEV)); + KUNIT_EXPECT_TRUE(test, t6040_sleep_keep_failed(false, -EIO)); + KUNIT_EXPECT_FALSE(test, t6040_sleep_keep_failed(false, -EBUSY)); + KUNIT_EXPECT_FALSE(test, t6040_sleep_keep_failed(false, 0)); + KUNIT_EXPECT_FALSE(test, t6040_sleep_keep_failed(false, APPLE_ATCPHY_SLEEP_RESET)); + KUNIT_EXPECT_FALSE(test, t6040_sleep_keep_failed(true, -ENODEV)); + KUNIT_EXPECT_FALSE(test, t6040_sleep_keep_failed(true, APPLE_ATCPHY_SLEEP_RESET)); +} + +/* + * The wake starts USB3 if it ran before the sleep. Not while the coprocessor + * waits for a power cycle after a stop it didn't acknowledge, until a sleep + * that powered the SoC down has been that power cycle. + */ +static void t6040_sleep_wake_usb3_test(struct kunit *test) +{ + struct t6040_usb2_phy *tphy = t6040_sleep_test_phy(test); + + KUNIT_EXPECT_FALSE(test, t6040_sleep_wake_usb3(tphy)); + tphy->sleep_usb3 = true; + KUNIT_EXPECT_TRUE(test, t6040_sleep_wake_usb3(tphy)); + t6040_recovery_power_down(&tphy->recovery, true, false); + KUNIT_EXPECT_FALSE(test, t6040_sleep_wake_usb3(tphy)); + t6040_recovery_power_lost(&tphy->recovery); + KUNIT_EXPECT_TRUE(test, t6040_sleep_wake_usb3(tphy)); +} + +/* + * A sleep that powered the SoC down has reset the USB2 PHY and power-cycled + * the coprocessor: a power-down that didn't land before it and a power cycle + * that was pending are behind the port, which sleeps in place again. + */ +static void t6040_sleep_power_lost_test(struct kunit *test) +{ + struct t6040_usb2_phy *tphy = t6040_sleep_test_phy(test); + + tphy->shutdown_failed = true; + tphy->sleep_usb3 = true; + t6040_recovery_power_down(&tphy->recovery, true, false); + KUNIT_EXPECT_FALSE(test, t6040_can_sleep_in_place(tphy)); + KUNIT_EXPECT_FALSE(test, t6040_sleep_wake_usb3(tphy)); + t6040_power_lost(tphy); + KUNIT_EXPECT_FALSE(test, tphy->shutdown_failed); + KUNIT_EXPECT_TRUE(test, t6040_can_sleep_in_place(tphy)); + KUNIT_EXPECT_TRUE(test, t6040_sleep_wake_usb3(tphy)); +} + +static struct kunit_case t6040_sleep_test_cases[] = { + KUNIT_CASE(t6040_sleep_usb_test), + KUNIT_CASE(t6040_sleep_no_host_test), + KUNIT_CASE(t6040_sleep_other_modes_test), + KUNIT_CASE(t6040_sleep_failed_test), + KUNIT_CASE(t6040_sleep_misses_test), + KUNIT_CASE(t6040_sleep_keep_failed_test), + KUNIT_CASE(t6040_sleep_wake_usb3_test), + KUNIT_CASE(t6040_sleep_power_lost_test), + {} +}; + +static struct kunit_suite t6040_sleep_test_suite = { + .name = "phy-apple-t6040-sleep", + .test_cases = t6040_sleep_test_cases, +}; + +kunit_test_suites(&t6040_recovery_test_suite, &t6040_sleep_test_suite); diff --git a/drivers/phy/apple/t6040-usb2.c b/drivers/phy/apple/t6040-usb2.c index 50d82116aac2d..08c36aed331d3 100644 --- a/drivers/phy/apple/t6040-usb2.c +++ b/drivers/phy/apple/t6040-usb2.c @@ -41,18 +41,23 @@ #include #include #include +#include +#include #include #include +#include #include #include #include #include #include +#include +#include #include #include #include #include -#include +#include #define T6040_USB2_USBCTL 0x00 #define T6040_USB2_USBCTL_MODE GENMASK(2, 0) @@ -124,10 +129,8 @@ #define T6040_CORE_CMN_OVRD_SLEEP_B_BIG BIT(0) #define T6040_CORE_CMN_OVRD_SLEEP_B_SML BIT(2) #define T6040_CORE_CMN_OVRD_CLAMP BIT(4) -#define T6040_CORE_LANE_RX 0x10 /* CFG1_TOP */ +#define T6040_CORE_CFG1_TOP 0x10 /* ACIOPHY_TOP_BLK_CFG1_TOP: per-lane RX power */ #define T6040_CORE_SLP_CTRL 0x120 /* CFG_SLP_CTRL */ -#define T6040_CORE_LN0_RX_TOP_CIO 0x935c -#define T6040_CORE_LN1_RX_TOP_CIO 0x1935c #define T6040_CORE_CMN_BIAS 0xa00 /* AUS40CMN_SHM CMN_BIAS_REG */ #define T6040_CORE_CMN_BIAS_BGBIAS_EN_OV BIT(1) #define T6040_CORE_UC_CTRL 0x260 /* ACIOPHY_TOP_BLK_UC_CTRL_REG */ @@ -175,6 +178,10 @@ #define T6040_CPU_MSG_POWER_DOWN 0x0100000000000000ULL #define T6040_RECOVERY_RETRIES 1 +#define T6040_RECOVERY_PM_RETRIES 5 +#define T6040_RECOVERY_PM_BACKOFF_US 2000 +/* DP's power-on waits this long for the wake of a host kept through a sleep */ +#define T6040_SLEEP_DP_WAIT_MS 2000 static unsigned int recovery_dwell_ms = 50; module_param(recovery_dwell_ms, uint, 0644); @@ -214,6 +221,44 @@ static void t6040_recovery_complete(struct t6040_recovery_state *state) state->retries = 0; } +/* + * The SoC lost power in a sleep, and the coprocessor and both of its power + * domains with it: that is the power cycle a stop that wasn't clean asks for. + */ +static void t6040_recovery_power_lost(struct t6040_recovery_state *state) +{ + state->cycle_pending = false; +} + +enum t6040_recovery_pm_action { + T6040_RECOVERY_PM_SUSPEND, + T6040_RECOVERY_PM_RETRY, + T6040_RECOVERY_PM_FALLBACK, +}; + +static enum t6040_recovery_pm_action +t6040_recovery_pm_action(int usage_count, int child_count, + unsigned int attempt, unsigned int retries) +{ + if (!usage_count && !child_count) + return T6040_RECOVERY_PM_SUSPEND; + if (attempt < retries) + return T6040_RECOVERY_PM_RETRY; + return T6040_RECOVERY_PM_FALLBACK; +} + +/* + * How often the recovery waits for the mailbox's users to go. Never when the + * PHY is one of them: forced into suspend while it is still in use, by a phy + * that is powered on or by the controller's reset, it stays a consumer of its + * suppliers, and the mailbox keeps the reference of that device link however + * long the recovery waits. + */ +static unsigned int t6040_recovery_pm_retries(bool phy_in_use) +{ + return phy_in_use ? 0 : T6040_RECOVERY_PM_RETRIES; +} + enum t6040_usb3_region { T6040_USB3_AXI2AF, T6040_USB3_AXI2AF_LIOA, @@ -239,24 +284,34 @@ static const char * const t6040_usb3_region_names[T6040_USB3_NR_REGIONS] = { * The ADT tables AppleT6040TypeCPhy applies at every USB3 power-on (mode 4), * in its order; the device tree carries them per port, as calibration. */ +enum t6040_usb3_tunable { + T6040_USB3_TUNABLE_AXI2AF, + T6040_USB3_TUNABLE_FABRIC, + T6040_USB3_TUNABLE_COMMON_CFG, + T6040_USB3_TUNABLE_AXI2AF_LIOA, + T6040_USB3_TUNABLE_LANE0, + T6040_USB3_TUNABLE_LANE1, + T6040_USB3_TUNABLE_AUS40CMN_SHM, + T6040_USB3_TUNABLE_AUSPLL, + T6040_USB3_TUNABLE_CIO4PLL, + T6040_USB3_NR_TUNABLES, +}; + static const struct { const char *prop; enum t6040_usb3_region region; -} t6040_usb3_tunable_list[] = { - { "apple,tunable-axi2af", T6040_USB3_AXI2AF }, - { "apple,tunable-fabric", T6040_USB3_FABRIC }, - { "apple,tunable-common-cfg", T6040_USB3_COMMON_CFG }, - { "apple,tunable-axi2af-lioa", T6040_USB3_AXI2AF_LIOA }, - { "apple,tunable-lane0", T6040_USB3_CORE }, - { "apple,tunable-lane1", T6040_USB3_CORE }, - { "apple,tunable-aus40cmn-shm", T6040_USB3_CORE }, - { "apple,tunable-auspll", T6040_USB3_CORE }, - { "apple,tunable-cio4pll", T6040_USB3_CORE }, +} t6040_usb3_tunable_list[T6040_USB3_NR_TUNABLES] = { + [T6040_USB3_TUNABLE_AXI2AF] = { "apple,tunable-axi2af", T6040_USB3_AXI2AF }, + [T6040_USB3_TUNABLE_FABRIC] = { "apple,tunable-fabric", T6040_USB3_FABRIC }, + [T6040_USB3_TUNABLE_COMMON_CFG] = { "apple,tunable-common-cfg", T6040_USB3_COMMON_CFG }, + [T6040_USB3_TUNABLE_AXI2AF_LIOA] = { "apple,tunable-axi2af-lioa", T6040_USB3_AXI2AF_LIOA }, + [T6040_USB3_TUNABLE_LANE0] = { "apple,tunable-lane0", T6040_USB3_CORE }, + [T6040_USB3_TUNABLE_LANE1] = { "apple,tunable-lane1", T6040_USB3_CORE }, + [T6040_USB3_TUNABLE_AUS40CMN_SHM] = { "apple,tunable-aus40cmn-shm", T6040_USB3_CORE }, + [T6040_USB3_TUNABLE_AUSPLL] = { "apple,tunable-auspll", T6040_USB3_CORE }, + [T6040_USB3_TUNABLE_CIO4PLL] = { "apple,tunable-cio4pll", T6040_USB3_CORE }, }; -#define T6040_USB3_TUNABLE_AUSPLL 7 -#define T6040_USB3_NR_TUNABLES ARRAY_SIZE(t6040_usb3_tunable_list) - struct t6040_usb2_phy { struct device *dev; void __iomem *usb2; @@ -281,6 +336,8 @@ struct t6040_usb2_phy { /* ADT tunable_USB2PHY_HOST, applied after the signal setup. */ struct apple_tunable *tunable_usb2_host; struct apple_rtkit *rtk; + struct notifier_block pd_nb; /* keeps the power domain on under a running coprocessor */ + bool fw_live; /* the coprocessor runs; pd_nb reads it under the domain's lock */ struct device *mbox_dev; struct completion app_reply; struct t6040_recovery_state recovery; @@ -292,7 +349,6 @@ struct t6040_usb2_phy { bool usb3_pipe; /* dwc3 asked for the PIPE switch this power cycle */ bool usb3_lent; /* DP mode took the lanes of this Type-C port */ bool pipe_switched; /* the controller's PIPE runs on this PHY */ - bool cio_up; /* the lanes run USB4/Thunderbolt */ /* DisplayPort; dp_ok is false when the DT lacks a DP route */ struct phy *dp; @@ -305,6 +361,28 @@ struct t6040_usb2_phy { bool dp_mode_flip; bool dp_flip; /* plug orientation reversed for the DP lanes */ int dp_rate; /* link rate index of the running link sequence, -1 = off */ + + /* CIO (USB4/Thunderbolt); cio_ok is false unless the DT has the CIO tables */ + struct apple_tunable *cio_tunables[5]; + bool cio_ok; + bool cio_up; /* PHY in mode 2, holds a runtime PM reference */ + bool pipe_tunneled; /* the controller's PIPE runs on the USB4 router */ + u8 tun_rate[2]; /* DP tunnel link rate per DP IN adapter, 0 = off */ + bool tun_pre; /* switch 1 is held for DP IN 1 alone */ + bool tun_pll; /* AUSPLL programmed for DP tunnels */ + bool router_running; /* the USB4 router runs on this port's CIO link */ + bool usb3_tunnel_req; /* the router carries USB3 for the controller */ + + /* A sleep with the host kept, see apple_t6040_atcphy_sleep_prepare() */ + bool sleep_armed; /* taken on by the prepare, until the finish */ + bool sleep_down; /* USB2 off and USB3 asleep for it, behind their consumers */ + bool sleep_reset; /* the controller held in reset for it, behind its consumer */ + bool sleep_usb3; /* USB3 ran at a sleep of this host, and comes back with USB2 */ + bool sleep_waking; /* in that wake: no power cycle of the PHY's domains */ + bool sleep_back; /* the system resumes and that wake is still to run */ + wait_queue_head_t sleep_wq; /* DP's power-on waits here for sleep_back to end */ + bool sleep_no_keep; /* keeping the host failed: it stops for the next sleep */ + int sleep_ret; /* what the wake has come to, for the controller's glue */ }; static inline void t6040_usb2_mask32(void __iomem *reg, u32 mask, u32 set) @@ -360,19 +438,12 @@ static u32 t6040_usb2_sig_want(struct t6040_usb2_phy *tphy) return want & T6040_USB2_SIG_INIT; } -static int t6040_usb2_power_on(struct phy *phy) +/* The USB2 host sequence itself. A field that did not take leaves the PHY failed. */ +static int t6040_usb2_on_locked(struct t6040_usb2_phy *tphy) { - struct t6040_usb2_phy *tphy = phy_get_drvdata(phy); u32 status, ctl, sig, tune, mode, sig_want; - guard(mutex)(&tphy->lock); - if (tphy->shutdown_failed) - return -EIO; - - if (tphy->mode != PHY_MODE_USB_HOST) - return -EINVAL; - if (tphy->powered) - return 0; + lockdep_assert_held(&tphy->lock); /* * eUSB2 PHY init, in this exact order. @@ -449,7 +520,6 @@ static int t6040_usb2_power_on(struct phy *phy) !(status & T6040_USB2_EVT_CTL_EN)) goto failed; - tphy->powered = true; dev_info(tphy->dev, "USB2 host sequence complete\n"); return 0; failed: @@ -457,21 +527,42 @@ static int t6040_usb2_power_on(struct phy *phy) return -EIO; } +static int t6040_usb2_power_on(struct phy *phy) +{ + struct t6040_usb2_phy *tphy = phy_get_drvdata(phy); + int ret; + + guard(mutex)(&tphy->lock); + if (tphy->shutdown_failed) + return -EIO; + + if (tphy->mode != PHY_MODE_USB_HOST) + return -EINVAL; + if (tphy->powered) + return 0; + + ret = t6040_usb2_on_locked(tphy); + if (ret) + return ret; + tphy->powered = true; + /* + * The controller's reset released and USB2 came up, which is all that + * keeping a host through a sleep needs to work again at its wake. + */ + tphy->sleep_no_keep = false; + return 0; +} + /* * Power the PHY down: port reset (held 5 ms), USBCTL mode 4, SIDDQ, PHY * reset, then gate the APB and reference clocks. The host controller must be * halted first; this is not a substitute for an xHCI halt. */ -static int t6040_usb2_power_off(struct phy *phy) +static int t6040_usb2_off_locked(struct t6040_usb2_phy *tphy) { - struct t6040_usb2_phy *tphy = phy_get_drvdata(phy); u32 ctl, mode, tune; - guard(mutex)(&tphy->lock); - if (tphy->shutdown_failed) - return -EIO; - if (!tphy->powered) - return 0; + lockdep_assert_held(&tphy->lock); tphy->shutdown_failed = true; if (!(readl(tphy->usb2 + T6040_USB2_CTL) & T6040_USB2_CTL_PORT_RESET)) { @@ -511,16 +602,47 @@ static int t6040_usb2_power_off(struct phy *phy) T6040_USB2_MISCTUNE_REFCLK_GATE_OFF)) return -EIO; tphy->shutdown_failed = false; + return 0; +} + +/* The wake of a sleep with the host kept has run, or won't: DP stops waiting for it. */ +static void t6040_sleep_back_done(struct t6040_usb2_phy *tphy) +{ + lockdep_assert_held(&tphy->lock); + if (!tphy->sleep_back) + return; + WRITE_ONCE(tphy->sleep_back, false); + wake_up_all(&tphy->sleep_wq); +} + +static int t6040_usb2_power_off(struct phy *phy) +{ + struct t6040_usb2_phy *tphy = phy_get_drvdata(phy); + int ret; + + guard(mutex)(&tphy->lock); + if (tphy->shutdown_failed) + return -EIO; + if (!tphy->powered) + return 0; + + /* Also the stop of a host whose PHY a sleep left down: the sequence only repeats. */ + ret = t6040_usb2_off_locked(tphy); + if (ret) + return ret; tphy->powered = false; + tphy->sleep_down = false; + t6040_sleep_back_done(tphy); return 0; } -static const struct phy_ops t6040_usb2_phy_ops = { +const struct phy_ops apple_t6040_usb2_phy_ops = { .owner = THIS_MODULE, .set_mode = t6040_usb2_set_mode, .power_on = t6040_usb2_power_on, .power_off = t6040_usb2_power_off, }; +EXPORT_SYMBOL_GPL(apple_t6040_usb2_phy_ops); /* ---- USB3 ---- */ @@ -565,30 +687,43 @@ static int t6040_usb3_wake(struct t6040_usb2_phy *tphy) return 0; } +/* The fixed DPTX tunable writes (apply_dptx_tunables, shared with USB3). */ +static const struct { u32 off, clear, set; } t6040_dptx_tunables[] = { + { 0x00800, 0x000003f0, 0x000001c0 }, /* AUS40CMN_DIG RCAL0 */ + { 0x00818, 0x0000f000, 0x00003000 }, /* AUS40CMN_DIG RCAL5 */ + { 0x03888, 0x000000ff, 0x00000027 }, /* CIO4PLL FREQDESCRIPTOR_0_C */ + { 0x28004, 0x00000038, 0x00000008 }, /* AUX_LDO_CTRL */ + { 0x03adc, 0x00003ff0, 0x00001760 }, /* CIO4PLL CFG_FRACN_FUNC */ + /* The lane entries, T6040_DPTX_LANE_TUNABLES of them, come last. */ + { 0x0e0cc, 0x01f80000, 0x00380000 }, /* LANE0 TX_EQ_CTRL_REG11 */ + { 0x1e0cc, 0x01f80000, 0x00380000 }, /* LANE1 TX_EQ_CTRL_REG11 */ + { 0x0d090, 0x01f80000, 0x00380000 }, /* LANE0 RX_CFG TX_EQ_CTRL_REG11 */ + { 0x1d090, 0x01f80000, 0x00380000 }, /* LANE1 RX_CFG TX_EQ_CTRL_REG11 */ +}; + +#define T6040_DPTX_LANE_TUNABLES 4 + +static void t6040_dp_fixed_tunables(void __iomem *core) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(t6040_dptx_tunables); i++) + t6040_usb2_mask32(core + t6040_dptx_tunables[i].off, t6040_dptx_tunables[i].clear, + t6040_dptx_tunables[i].set); +} + /* * B1: the tables, then apply_dptx_tunables' fixed writes (atc-phy reg[] offsets * relative to the core) and AUSPLL_CORE once more. */ static void t6040_usb3_tunables(struct t6040_usb2_phy *tphy) { - static const struct { u32 off, clear, set; } dptx[] = { - { 0x00800, 0x000003f0, 0x000001c0 }, /* AUS40CMN_DIG RCAL0 */ - { 0x00818, 0x0000f000, 0x00003000 }, /* AUS40CMN_DIG RCAL5 */ - { 0x03888, 0x000000ff, 0x00000027 }, /* CIO4PLL FREQDESCRIPTOR_0_C */ - { 0x28004, 0x00000038, 0x00000008 }, /* AUX_LDO_CTRL */ - { 0x03adc, 0x00003ff0, 0x00001760 }, /* CIO4PLL CFG_FRACN_FUNC */ - { 0x0e0cc, 0x01f80000, 0x00380000 }, /* LANE0 TX_EQ_CTRL_REG11 */ - { 0x1e0cc, 0x01f80000, 0x00380000 }, /* LANE1 TX_EQ_CTRL_REG11 */ - { 0x0d090, 0x01f80000, 0x00380000 }, /* LANE0 RX_CFG TX_EQ_CTRL_REG11 */ - { 0x1d090, 0x01f80000, 0x00380000 }, /* LANE1 RX_CFG TX_EQ_CTRL_REG11 */ - }; size_t i; for (i = 0; i < T6040_USB3_NR_TUNABLES; i++) apple_tunable_apply(tphy->regs[t6040_usb3_tunable_list[i].region], tphy->tunables[i]); - for (i = 0; i < ARRAY_SIZE(dptx); i++) - t6040_usb2_mask32(t6040_core(tphy) + dptx[i].off, dptx[i].clear, dptx[i].set); + t6040_dp_fixed_tunables(t6040_core(tphy)); apple_tunable_apply(t6040_core(tphy), tphy->tunables[T6040_USB3_TUNABLE_AUSPLL]); } @@ -671,25 +806,24 @@ static int t6040_usb3_cpu_start(struct t6040_usb2_phy *tphy) dev_err(tphy->dev, "USB3: the PHY coprocessor runs without an RTKit session\n"); return -EBUSY; } + /* From here to the end of its stop the coprocessor's power domains stay on. */ + WRITE_ONCE(tphy->fw_live, true); writel(T6040_CPU_RUN_START, cpu + T6040_CPU_RUN); usleep_range(5000, 6000); return apple_rtkit_boot(tphy->rtk); } /* - * C: runFirmware. macOS starts the coprocessor on every attach and stops it on - * every detach; each start makes the firmware tune the PHY. Then release the - * PHY reset (D). + * Part C of the PHY init from the delay after the lane swap on, for the modes + * that run the PHY firmware (USB3 and CIO): the firmware start, the tune-done + * poll and the reset release (D). @what and @lane only label the log. */ -static int t6040_usb3_run_firmware(struct t6040_usb2_phy *tphy, bool usb, bool swap) +static int t6040_phy_run_firmware(struct t6040_usb2_phy *tphy, const char *what, int lane) { void __iomem *cpu = t6040_cpu(tphy); u32 uc_before; int ret; - if (usb) - t6040_usb2_mask32(t6040_cfg(tphy) + T6040_CFG_SPARE0, T6040_CFG_SPARE0_USB_LN_SWAP, - swap ? T6040_CFG_SPARE0_USB_LN_SWAP : 0); fsleep(20); t6040_usb2_set32(cpu + T6040_CPU_IDLE_CTRL, T6040_CPU_IDLE_CTRL_EN); fsleep(10); @@ -698,27 +832,39 @@ static int t6040_usb3_run_firmware(struct t6040_usb2_phy *tphy, bool usb, bool s if (!tphy->rtk || !apple_rtkit_is_running(tphy->rtk)) { ret = t6040_usb3_cpu_start(tphy); if (ret) { - dev_err(tphy->dev, "USB3: the PHY coprocessor was not started: %d (run %#x, idle %#x, status %#x)\n", - ret, readl(cpu + T6040_CPU_RUN), + dev_err(tphy->dev, "%s: the PHY coprocessor was not started: %d (run %#x, idle %#x, status %#x)\n", + what, ret, readl(cpu + T6040_CPU_RUN), readl(cpu + T6040_CPU_IDLE_CTRL), readl(cpu + T6040_CPU_STATUS)); return ret; } } ret = t6040_usb3_wait_tuned(tphy); if (ret) { - dev_err(tphy->dev, "USB3: the PHY coprocessor did not tune the PHY (uc %#x -> %#x, idle %#x, status %#x)\n", - uc_before, readl(t6040_core(tphy) + T6040_CORE_UC_CTRL), + dev_err(tphy->dev, "%s: the PHY coprocessor did not tune the PHY (uc %#x -> %#x, idle %#x, status %#x)\n", + what, uc_before, readl(t6040_core(tphy) + T6040_CORE_UC_CTRL), readl(cpu + T6040_CPU_IDLE_CTRL), readl(cpu + T6040_CPU_STATUS)); return ret; } - dev_info(tphy->dev, "USB3 PHY tuned: lane %d, coprocessor started, uc %#x -> %#x\n", swap, - uc_before, readl(t6040_core(tphy) + T6040_CORE_UC_CTRL)); + dev_info(tphy->dev, "%s PHY tuned: lane %d, coprocessor started, uc %#x -> %#x\n", what, + lane, uc_before, readl(t6040_core(tphy) + T6040_CORE_UC_CTRL)); t6040_usb2_set32(t6040_cfg(tphy) + T6040_CFG_CTRL, T6040_CFG_CTRL_PHY_RESET_N); fsleep(100); return 0; } +/* + * C: the firmware start. The coprocessor is started on every attach and stopped on + * every detach; each start makes the firmware tune the PHY. Then release the + * PHY reset (D). + */ +static int t6040_usb3_run_firmware(struct t6040_usb2_phy *tphy, bool swap) +{ + t6040_usb2_mask32(t6040_cfg(tphy) + T6040_CFG_SPARE0, T6040_CFG_SPARE0_USB_LN_SWAP, + swap ? T6040_CFG_SPARE0_USB_LN_SWAP : 0); + return t6040_phy_run_firmware(tphy, "USB3", swap); +} + /* * Detach (RTKit power state 0): Power Down, RTKit AP then IOP to idle, * wait up to 10 ms for the CPU to idle, STOP. The RTKit transport stays stopped @@ -767,6 +913,8 @@ static void t6040_usb3_firmware_power_down(struct t6040_usb2_phy *tphy) /* Leave the transport stopped until the next firmware start. Free drains RX. */ t6040_rtkit_stop_transport(tphy); + /* Stopped, or to be power-cycled: either way its domains may go off again. */ + WRITE_ONCE(tphy->fw_live, false); /* Power Down is asynchronous; never wait solely for its application reply. */ acknowledged = completion_done(&tphy->app_reply); if (!acknowledged) @@ -817,6 +965,63 @@ static int t6040_common_dp_force_resume(struct t6040_usb2_phy *tphy) return 0; } +/* + * The RTKit transport owns one phymxwrap runtime reference while its mailbox + * is started. apple_rtkit_free() releases it before this function runs. Give + * an unrelated user or active child a bounded interval to become idle, but do + * not steal its reference: recovery can still make a clean firmware start + * without cycling a domain which is legitimately shared. + * + * Return 0 after a real suspend, 1 when the caller should use that normal-start + * fallback, and a negative error for a provider failure unrelated to a holder. + * @retries is how often to wait for a holder, see t6040_recovery_pm_retries(). + */ +static int t6040_usb3_suspend_mbox(struct t6040_usb2_phy *tphy, unsigned int retries) +{ + struct device *dev = tphy->mbox_dev; + unsigned int attempt; + int children, ret = -EBUSY, usage; + + for (attempt = 0; attempt <= retries; attempt++) { + enum t6040_recovery_pm_action action; + + pm_runtime_barrier(dev); + usage = atomic_read(&dev->power.usage_count); + children = atomic_read(&dev->power.child_count); + dev_dbg(tphy->dev, + "USB3: phymxwrap recovery attempt %u: usage_count=%d child_count=%d\n", + attempt, usage, children); + + action = t6040_recovery_pm_action(usage, children, attempt, retries); + if (action == T6040_RECOVERY_PM_SUSPEND) { + ret = pm_runtime_suspend(dev); + if (ret >= 0 && pm_runtime_status_suspended(dev)) + return 0; + if (ret >= 0) + ret = -EBUSY; + if (ret != -EAGAIN && ret != -EBUSY && ret != -EACCES) + return ret; + } else { + ret = usage ? -EAGAIN : -EBUSY; + if (action == T6040_RECOVERY_PM_FALLBACK) + break; + } + + if (attempt < retries) { + unsigned int delay = T6040_RECOVERY_PM_BACKOFF_US << attempt; + + usleep_range(delay, delay + T6040_RECOVERY_PM_BACKOFF_US); + } + } + + usage = atomic_read(&dev->power.usage_count); + children = atomic_read(&dev->power.child_count); + dev_warn(tphy->dev, + "USB3: phymxwrap remains active (usage_count=%d child_count=%d, suspend %d); continuing with normal firmware start\n", + usage, children, ret); + return 1; +} + /* * Drop both domains with RTKit fully detached. phymxwrap is the mailbox * provider's runtime domain; common_dp is this device's runtime domain. Force @@ -827,24 +1032,25 @@ static int t6040_common_dp_force_resume(struct t6040_usb2_phy *tphy) static int t6040_usb3_hard_recover(struct t6040_usb2_phy *tphy) { unsigned int dwell = max(recovery_dwell_ms, 1U); + unsigned int retries; int ret, resume_ret; - if (WARN_ON(tphy->rtk)) - t6040_rtkit_stop_transport(tphy); + /* This also releases the mailbox provider reference synchronously. */ + t6040_rtkit_stop_transport(tphy); ret = pm_runtime_force_suspend(tphy->dev); if (ret < 0) return ret; - pm_runtime_barrier(tphy->mbox_dev); - ret = pm_runtime_suspend(tphy->mbox_dev); - if (ret < 0 && ret != -EACCES) { + /* Left active by that: still in use, and so still the mailbox's consumer. */ + retries = t6040_recovery_pm_retries(!pm_runtime_status_suspended(tphy->dev)); + ret = t6040_usb3_suspend_mbox(tphy, retries); + if (ret < 0) { dev_err(tphy->dev, "USB3: phymxwrap suspend failed: %d\n", ret); goto resume_phy; } - if (!pm_runtime_status_suspended(tphy->mbox_dev)) { - ret = -EBUSY; - dev_err(tphy->dev, "USB3: phymxwrap did not power off\n"); + if (ret > 0) { + ret = 0; goto resume_phy; } @@ -915,15 +1121,15 @@ static void t6040_usb3_bist_off(struct t6040_usb2_phy *tphy) } /* - * aciophy_phy_shutdown_partA: the PHY's USB link side down (the same BIST - * handshake as the PIPE switch), both TX lanes, then the common block. + * Part A of the PHY shutdown for every firmware mode: both TX lanes + * powered down, then the common block shut down. */ -/* aciophy_tx_powerdown(0), (1), then aciophy_cmn_shutdown: USB and CIO alike. */ -static void t6040_phy_tx_cmn_down(struct t6040_usb2_phy *tphy) +static void t6040_phy_tx_cmn_shutdown(struct t6040_usb2_phy *tphy) { void __iomem *core = t6040_core(tphy); unsigned int lane; + /* The TX power-down of lane 0, then of lane 1 */ for (lane = 0; lane < 2; lane++) { t6040_usb2_set32(core + T6040_CORE_SLP_CTRL, 0x100 << lane); udelay(1); @@ -944,6 +1150,10 @@ static void t6040_phy_tx_cmn_down(struct t6040_usb2_phy *tphy) udelay(1); } +/* + * Part A of the PHY shutdown: the PHY's USB link side down (the same BIST + * handshake as the PIPE switch), both TX lanes, then the common block. + */ static void t6040_usb3_shutdown_lanes(struct t6040_usb2_phy *tphy) { void __iomem *bist = t6040_bist(tphy); @@ -959,8 +1169,8 @@ static void t6040_usb3_shutdown_lanes(struct t6040_usb2_phy *tphy) t6040_usb3_bist_wait(tphy, T6040_BIST_STATUS_PHY_2, true, 1000); t6040_usb2_clear32(bist + T6040_BIST_CFG0, BIT(0)); - t6040_phy_tx_cmn_down(tphy); - fsleep(200); + t6040_phy_tx_cmn_shutdown(tphy); + fsleep(200); /* the PHY shutdown waits 200 us after part A */ } static int t6040_usb3_pipe_lock(struct t6040_usb2_phy *tphy, unsigned int ms) @@ -1071,11 +1281,22 @@ static void t6040_usb3_up_locked(struct t6040_usb2_phy *tphy) lockdep_assert_held(&tphy->lock); if (!tphy->usb3_ok || tphy->usb3_up) return; - /* DP and CIO own all four lanes; USB3 comes back in t6040_usb3_reclaim_locked(). */ - if (tphy->dp_up || tphy->cio_up) { + /* Down for a sleep with the host kept: USB3 comes back with the wake, after USB2. */ + if (tphy->sleep_down) { + dev_dbg(tphy->dev, "USB3: the PHY is down for a sleep, not started\n"); + return; + } + /* DP mode owns all four lanes; USB3 comes back in t6040_usb3_reclaim_locked(). */ + if (tphy->dp_up) { dev_dbg(tphy->dev, "USB3: lanes held by DP, USB2 only for now\n"); return; } + /* CIO owns both lanes; USB3 comes back in t6040_usb3_reclaim_locked() after it. */ + if (tphy->cio_up) { + dev_dbg(tphy->dev, "USB3: lanes held by CIO, USB2 only for now\n"); + tphy->usb3_lent = true; + return; + } switch (tphy->orientation) { case TYPEC_ORIENTATION_NORMAL: @@ -1105,7 +1326,7 @@ static void t6040_usb3_up_locked(struct t6040_usb2_phy *tphy) goto sleep; /* the coprocessor block may be unclocked: never touch it */ t6040_usb3_tunables(tphy); t6040_usb3_lanes(tphy, swap); - ret = t6040_usb3_run_firmware(tphy, true, swap); + ret = t6040_usb3_run_firmware(tphy, swap); if (ret) goto retry_or_fail; @@ -1118,7 +1339,11 @@ static void t6040_usb3_up_locked(struct t6040_usb2_phy *tphy) t6040_usb3_shutdown_lanes(tphy); t6040_usb3_firmware_power_down(tphy); t6040_usb3_sleep(tphy); - if (t6040_recovery_retry(&tphy->recovery)) { + /* + * The retry power-cycles the PHY's domains, which don't go off under a system + * resume: a wake with the host kept leaves the port at USB2 instead. + */ + if (t6040_recovery_retry(&tphy->recovery) && !tphy->sleep_waking) { cycle = true; goto retry; } @@ -1131,6 +1356,8 @@ static void t6040_usb3_up_locked(struct t6040_usb2_phy *tphy) dev_warn(tphy->dev, "USB3 PHY bring-up failed (%d), USB2 only\n", ret); } +static void t6040_cio_pipe_update_locked(struct t6040_usb2_phy *tphy); + static int t6040_usb3_power_on(struct phy *phy) { struct t6040_usb2_phy *tphy = phy_get_drvdata(phy); @@ -1148,6 +1375,11 @@ static void t6040_usb3_down_locked(struct t6040_usb2_phy *tphy) t6040_usb3_pipe_park(tphy); tphy->pipe_switched = false; } + /* Off the USB4 router: the park from there uses the same writes. */ + if (tphy->pipe_tunneled) { + t6040_usb3_pipe_park(tphy); + tphy->pipe_tunneled = false; + } /* DWC3 can drop its role before the DP fabric has released pin D. */ if (tphy->dp_up && tphy->dp_mode_two_lanes) return; @@ -1172,6 +1404,8 @@ static int t6040_usb3_power_off(struct phy *phy) guard(mutex)(&tphy->lock); tphy->usb3_on = false; tphy->usb3_pipe = false; + /* The host stops: what its next start brings up is what a later wake brings back. */ + tphy->sleep_usb3 = false; t6040_usb3_down_locked(tphy); return 0; } @@ -1190,6 +1424,10 @@ static int t6040_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode) guard(mutex)(&tphy->lock); tphy->usb3_pipe = true; + if (tphy->cio_up) { + t6040_cio_pipe_update_locked(tphy); + return 0; + } if (!tphy->usb3_up || tphy->pipe_switched) return 0; t6040_usb3_pipe_switch(tphy); @@ -1236,26 +1474,6 @@ static void t6040_usb3_switch_unregister(void *sw) * the two-lane group link sequences below. */ -/* The fixed DPTX tunable writes (apply_dptx_tunables, shared with USB3). */ -static void t6040_dp_fixed_tunables(void __iomem *core) -{ - static const struct { u32 off, clear, set; } dptx[] = { - { 0x00800, 0x000003f0, 0x000001c0 }, /* AUS40CMN_DIG RCAL0 */ - { 0x00818, 0x0000f000, 0x00003000 }, /* AUS40CMN_DIG RCAL5 */ - { 0x03888, 0x000000ff, 0x00000027 }, /* CIO4PLL FREQDESCRIPTOR_0_C */ - { 0x28004, 0x00000038, 0x00000008 }, /* AUX_LDO_CTRL */ - { 0x03adc, 0x00003ff0, 0x00001760 }, /* CIO4PLL CFG_FRACN_FUNC */ - { 0x0e0cc, 0x01f80000, 0x00380000 }, /* LANE0 TX_EQ_CTRL_REG11 */ - { 0x1e0cc, 0x01f80000, 0x00380000 }, /* LANE1 TX_EQ_CTRL_REG11 */ - { 0x0d090, 0x01f80000, 0x00380000 }, /* LANE0 RX_CFG TX_EQ_CTRL_REG11 */ - { 0x1d090, 0x01f80000, 0x00380000 }, /* LANE1 RX_CFG TX_EQ_CTRL_REG11 */ - }; - size_t i; - - for (i = 0; i < ARRAY_SIZE(dptx); i++) - t6040_usb2_mask32(core + dptx[i].off, dptx[i].clear, dptx[i].set); -} - /* aciophy_xbar_config for PHY mode 3 (DP on all four lanes). */ static void t6040_dp_xbar_config(struct t6040_usb2_phy *tphy, bool flip) { @@ -1375,8 +1593,8 @@ static bool t6040_dp_only(struct t6040_usb2_phy *tphy) return !tphy->usb3_ok; } -/* apply_tunables' generic tables: the first six of t6040_usb3_tunable_list. */ -#define T6040_DP_NR_GENERIC_TUNABLES 6 +/* apply_tunables' generic tables: those of t6040_usb3_tunable_list up to the lanes'. */ +#define T6040_DP_NR_GENERIC_TUNABLES (T6040_USB3_TUNABLE_LANE1 + 1) /* * slpctrl_init for DP on both lanes: the cmn_rm_ovrd/txrx_rm_ovrd releases of @@ -1522,6 +1740,25 @@ static int t6040_dp_seq_step(struct t6040_usb2_phy *tphy, const struct atc_op *s return ret; } +/* + * A lane's registers are back at their reset values each time the lane has been powered + * down: after one link at a rate, one DPMS cycle or one failed training, the lane tables + * of the ADT and the DPTX lane tunables were gone for every link that followed. Write them + * again once the link sequence has the lanes up. + */ +static void t6040_dp_lane_tunables(struct t6040_usb2_phy *tphy) +{ + void __iomem *core = t6040_core(tphy); + size_t i; + + for (i = T6040_USB3_TUNABLE_LANE0; i <= T6040_USB3_TUNABLE_LANE1; i++) + apple_tunable_apply(core, tphy->tunables[i]); + for (i = ARRAY_SIZE(t6040_dptx_tunables) - T6040_DPTX_LANE_TUNABLES; + i < ARRAY_SIZE(t6040_dptx_tunables); i++) + t6040_usb2_mask32(core + t6040_dptx_tunables[i].off, t6040_dptx_tunables[i].clear, + t6040_dptx_tunables[i].set); +} + /* Take the link down if it is up, then bring it up at rate index @idx (-1: off). */ static int t6040_dp_set_rate(struct t6040_usb2_phy *tphy, int idx) { @@ -1565,6 +1802,9 @@ static int t6040_dp_set_rate(struct t6040_usb2_phy *tphy, int idx) if (ret) return ret; } + /* Pin D leaves the lanes to USB3's firmware, which keeps them up. */ + if (!tphy->dp_mode_two_lanes) + t6040_dp_lane_tunables(tphy); ret = t6040_dp_configure_clks(tphy); if (ret) return ret; @@ -1615,229 +1855,116 @@ static void t6040_usb3_reclaim_locked(struct t6040_usb2_phy *tphy) dev_info(tphy->dev, "USB3 %s after DP\n", tphy->usb3_up ? "back" : "not restored"); } -/* ---- CIO: USB4 and Thunderbolt on both lanes ---- */ - -/* The ADT tables of the common set, then the CIO ones; no DPTX writes. */ -static void t6040_cio_tunables(struct t6040_usb2_phy *tphy) -{ - void __iomem *core = t6040_core(tphy); - size_t i; - - for (i = 0; i < T6040_USB3_NR_TUNABLES; i++) - apple_tunable_apply(tphy->regs[t6040_usb3_tunable_list[i].region], - tphy->tunables[i]); - /* tunable_LN0/1_RX_TOP_CIO_DFLT; UC_REGS_CIO and RX_EQ_CIO are empty. */ - t6040_usb2_clear32(core + T6040_CORE_LN0_RX_TOP_CIO, 0xe00); - t6040_usb2_clear32(core + T6040_CORE_LN1_RX_TOP_CIO, 0xe00); -} +/* ---- DP phy_ops ---- */ -/* aciophy_xbar_config, mode 2: orientation is one crossbar bit. */ -static void t6040_cio_xbar(struct t6040_usb2_phy *tphy, bool flip) +/* + * A port that is down for a sleep with its host kept, while the system + * resumes: the Type-C controller can report a display before the host's + * resume has brought the PHY back, one that came to a dock during the sleep + * or an adapter in the place of the USB device. DP waits for that wake, and + * then finds the port as on any running host: USB3 up for pin assignment D + * to share, or to lend its lanes. If the wake fails or doesn't come, DP goes + * on and gets what it gets on a port without a host. + * + * The wait runs in a work item, not under the PM core: the display's driver + * powers a port's DP phy on from its link work, and after a sleep it queues + * that from its own resume on a workqueue that isn't frozen, so the work can + * get here while devices still resume. What the Type-C controller reports + * comes from works that stay frozen until every device has resumed. + */ +static void t6040_dp_wait_host(struct t6040_usb2_phy *tphy) { - void __iomem *core = t6040_core(tphy); - - t6040_usb2_clear32(core + T6040_CORE_XBAR, 0xfc); - t6040_usb2_mask32(core + T6040_CORE_XBAR_LANES, 0x1f, flip ? 1 : 0); - t6040_usb2_mask32(core + T6040_CORE_XBAR, 0x3, 0x2); + if (!READ_ONCE(tphy->sleep_back)) + return; + if (!wait_event_timeout(tphy->sleep_wq, !READ_ONCE(tphy->sleep_back), + msecs_to_jiffies(T6040_SLEEP_DP_WAIT_MS))) + dev_warn(tphy->dev, "DP: the host's wake from the sleep did not come in %u ms\n", + T6040_SLEEP_DP_WAIT_MS); } -/* aciophy_slpctrl_init, mode 2: both lanes, whatever the orientation. */ -static void t6040_cio_lanes_on(struct t6040_usb2_phy *tphy) +static int t6040_dp_power_on(struct phy *phy) { - static const struct { u32 off, bit; bool set; } steps[] = { - { T6040_CORE_LANE_RX, 0x10, true }, { T6040_CORE_LANE_RX, 0x40, true }, - { T6040_CORE_LANE_RX, 0x1, true }, { T6040_CORE_LANE_RX, 0x4, true }, - { T6040_CORE_CMN_OVRD, 0x400, true }, { T6040_CORE_CMN_OVRD, 0x1000, true }, - { T6040_CORE_CMN_OVRD, 0x40, true }, { T6040_CORE_CMN_OVRD, 0x100, true }, - { T6040_CORE_CMN_OVRD, 0x4000, false }, { T6040_CORE_CMN_OVRD, 0x10000, true }, - { T6040_CORE_SLP_CTRL, 0x40, false }, { T6040_CORE_SLP_CTRL, 0x4, false }, - { T6040_CORE_SLP_CTRL, 0x400, false }, - }; - void __iomem *core = t6040_core(tphy); - unsigned int lane; - size_t i; + struct t6040_usb2_phy *tphy = phy_get_drvdata(phy); + int ret; - t6040_usb2_clear32(core + T6040_CORE_CMN_OVRD, 0x2a); - udelay(1); - for (lane = 0; lane < 2; lane++) { - for (i = 0; i < ARRAY_SIZE(steps); i++) { - if (steps[i].set) - t6040_usb2_set32(core + steps[i].off, steps[i].bit << lane); - else - t6040_usb2_clear32(core + steps[i].off, steps[i].bit << lane); - udelay(1); + t6040_dp_wait_host(tphy); + guard(mutex)(&tphy->lock); + if (!tphy->dp_ok || tphy->dp_up) + return 0; + /* The port carries USB4/Thunderbolt: DP reaches it only as a tunnel. */ + if (tphy->cio_up) + return -EBUSY; + /* Pin D shares the running PHY with USB3. Keep its PIPE and firmware. */ + if (tphy->dp_two_lanes) { + if (!tphy->usb3_ok || !tphy->dp_alt) + return -ENOLINK; + if (!tphy->usb3_up) + t6040_usb3_up_locked(tphy); + if (!tphy->usb3_up) { + if (tphy->sleep_down) + dev_info(tphy->dev, + "DP PHY pin D: the port is down for a sleep, no link\n"); + return -ENOLINK; } + /* USB3 startup already applied these DPTX tunables before firmware. */ + t6040_dp_aux_init(tphy); + tphy->dp_rate = -1; + tphy->dp_mode_two_lanes = true; + tphy->dp_mode_flip = tphy->dp_flip; + tphy->dp_up = true; + dev_info(tphy->dev, "DP PHY pin D: two lanes, USB3 retained, %s orientation\n", + tphy->dp_flip ? "reverse" : "normal"); + return 0; } -} - -/* aciophy_rx_powerdown(0), (1): CIO powers the receivers down, USB does not. */ -static void t6040_cio_rx_down(struct t6040_usb2_phy *tphy) -{ - void __iomem *core = t6040_core(tphy); - unsigned int lane; - - for (lane = 0; lane < 2; lane++) { - t6040_usb2_set32(core + T6040_CORE_CMN_OVRD, 0x4000 << lane); - udelay(1); - t6040_usb2_clear32(core + T6040_CORE_CMN_OVRD, 0x40 << lane); - udelay(1); - t6040_usb2_clear32(core + T6040_CORE_CMN_OVRD, 0x400 << lane); - udelay(1); - t6040_usb2_clear32(core + T6040_CORE_LANE_RX, 0x1 << lane); - udelay(1); - t6040_usb2_clear32(core + T6040_CORE_LANE_RX, 0x10 << lane); - udelay(1); + /* C/E and the fixed HDMI PHY hand all four lanes to DP. */ + if (!t6040_dp_only(tphy)) { + t6040_usb3_down_locked(tphy); + tphy->usb3_lent = true; + } + ret = t6040_dp_phy_init(tphy, tphy->dp_flip); + if (ret) { + dev_err(tphy->dev, "DP PHY init failed: %d\n", ret); + return ret; } + t6040_dp_mode_setup(tphy, tphy->dp_flip); + tphy->dp_rate = -1; + tphy->dp_mode_two_lanes = false; + tphy->dp_up = true; + dev_dbg(tphy->dev, "DP PHY mode set, lane map %s\n", + tphy->dp_flip ? "reversed" : "normal"); + return 0; } -/* The Type-C state that needs CIO: USB4 entered, or the Thunderbolt alt mode. */ -static bool t6040_cio_state(const struct typec_mux_state *state) +/* + * The PHY's close for DP on all four lanes (with a DP client: the DP + * transmitter's shutdown, the AUX shutdown, the PHY shutdown of mode 3): a link still up goes + * down, then both TX lanes and the common block (part A), and after its 200 us the CFG + * block returns to its boot state (part B; no coprocessor to stop in this mode). The AUX + * block has no step of its own here: it goes into reset and behind the clamp with the PHY. + * + * Without this the fixed DP PHY of the HDMI converter stays as it ran once it is powered + * off; a Type-C port's was put to sleep, with no step for a link still up. The sleep engine + * of an S2R stalled on a port left as it ran, and the next power-on programmed a PHY that + * had not been through its reset. + */ +static void t6040_dp_shutdown(struct t6040_usb2_phy *tphy) { - if (state->alt) - return state->alt->svid == USB_TYPEC_TBT_SID; - if (state->mode != TYPEC_MODE_USB4 || !state->data) - return false; - return FIELD_GET(EUDO_USB_MODE_MASK, - ((const struct enter_usb_data *)state->data)->eudo) == EUDO_USB_MODE_USB4; + int ret = t6040_dp_set_rate(tphy, -1); + + if (ret) + dev_warn(tphy->dev, "DP link not taken down before the PHY shutdown: %d\n", ret); + tphy->dp_rate = -1; + t6040_phy_tx_cmn_shutdown(tphy); + fsleep(200); + t6040_usb3_sleep(tphy); + dev_dbg(tphy->dev, "DP PHY shut down: cfg %#x/%#x\n", + readl(t6040_cfg(tphy) + T6040_CFG_CTRL), readl(t6040_cfg(tphy) + T6040_CFG_STATUS)); } -static int t6040_cio_up_locked(struct t6040_usb2_phy *tphy) +static int t6040_dp_power_off(struct phy *phy) { - bool cycle; - bool flip; - int ret; - - lockdep_assert_held(&tphy->lock); - if (!tphy->usb3_ok || tphy->dp_up) - return -EBUSY; - switch (tphy->orientation) { - case TYPEC_ORIENTATION_NORMAL: - flip = false; - break; - case TYPEC_ORIENTATION_REVERSE: - flip = true; - break; - default: - dev_warn(tphy->dev, "CIO: no plug orientation\n"); - return -EINVAL; - } - if (tphy->usb3_up || tphy->pipe_switched) - t6040_usb3_down_locked(tphy); - tphy->usb3_lent = tphy->usb3_on; - - ret = pm_runtime_resume_and_get(tphy->dev); - if (ret) - return ret; - - cycle = t6040_recovery_start(&tphy->recovery); -retry: - if (cycle) { - ret = t6040_usb3_hard_recover(tphy); - if (ret) - goto put; - } - - ret = t6040_usb3_wake(tphy); - if (ret) - goto sleep; /* the coprocessor block may be unclocked: never touch it */ - t6040_cio_tunables(tphy); - t6040_cio_xbar(tphy, flip); - t6040_cio_lanes_on(tphy); - ret = t6040_usb3_run_firmware(tphy, false, flip); - if (ret) { - t6040_phy_tx_cmn_down(tphy); - t6040_cio_rx_down(tphy); - fsleep(200); - t6040_usb3_firmware_power_down(tphy); - t6040_usb3_sleep(tphy); - if (t6040_recovery_retry(&tphy->recovery)) { - cycle = true; - goto retry; - } - goto put; - } - t6040_recovery_complete(&tphy->recovery); - tphy->cio_up = true; - dev_info(tphy->dev, "CIO PHY up, %s orientation\n", flip ? "reverse" : "normal"); - return 0; - -sleep: - t6040_usb3_sleep(tphy); -put: - pm_runtime_put(tphy->dev); - dev_warn(tphy->dev, "CIO PHY bring-up failed: %d\n", ret); - return ret; -} - -static void t6040_cio_down_locked(struct t6040_usb2_phy *tphy) -{ - lockdep_assert_held(&tphy->lock); - if (!tphy->cio_up) - return; - t6040_phy_tx_cmn_down(tphy); - t6040_cio_rx_down(tphy); - fsleep(200); - t6040_usb3_firmware_power_down(tphy); - t6040_usb3_sleep(tphy); - tphy->cio_up = false; - pm_runtime_put(tphy->dev); - dev_info(tphy->dev, "CIO PHY asleep\n"); - t6040_usb3_reclaim_locked(tphy); -} - -/* ---- DP phy_ops ---- */ - -static int t6040_dp_power_on(struct phy *phy) -{ - struct t6040_usb2_phy *tphy = phy_get_drvdata(phy); - int ret; - - guard(mutex)(&tphy->lock); - if (!tphy->dp_ok || tphy->dp_up) - return 0; - if (tphy->cio_up) - return -EBUSY; - /* Pin D shares the running PHY with USB3. Keep its PIPE and firmware. */ - if (tphy->dp_two_lanes) { - if (!tphy->usb3_ok || !tphy->dp_alt) - return -ENOLINK; - if (!tphy->usb3_up) - t6040_usb3_up_locked(tphy); - if (!tphy->usb3_up) - return -ENOLINK; - /* USB3 startup already applied these DPTX tunables before firmware. */ - t6040_dp_aux_init(tphy); - tphy->dp_rate = -1; - tphy->dp_mode_two_lanes = true; - tphy->dp_mode_flip = tphy->dp_flip; - tphy->dp_up = true; - dev_info(tphy->dev, "DP PHY pin D: two lanes, USB3 retained, %s orientation\n", - tphy->dp_flip ? "reverse" : "normal"); - return 0; - } - /* C/E and the fixed HDMI PHY hand all four lanes to DP. */ - if (!t6040_dp_only(tphy)) { - t6040_usb3_down_locked(tphy); - tphy->usb3_lent = true; - } - ret = t6040_dp_phy_init(tphy, tphy->dp_flip); - if (ret) { - dev_err(tphy->dev, "DP PHY init failed: %d\n", ret); - return ret; - } - t6040_dp_mode_setup(tphy, tphy->dp_flip); - tphy->dp_rate = -1; - tphy->dp_mode_two_lanes = false; - tphy->dp_up = true; - dev_dbg(tphy->dev, "DP PHY mode set, lane map %s\n", - tphy->dp_flip ? "reversed" : "normal"); - return 0; -} - -static int t6040_dp_power_off(struct phy *phy) -{ - struct t6040_usb2_phy *tphy = phy_get_drvdata(phy); - int ret = 0; + struct t6040_usb2_phy *tphy = phy_get_drvdata(phy); + int ret = 0; guard(mutex)(&tphy->lock); if (tphy->dp_up && tphy->dp_mode_two_lanes) { @@ -1852,9 +1979,8 @@ static int t6040_dp_power_off(struct phy *phy) * sleeping PHY, as on a fresh attach. Booting the coprocessor's * firmware for the first time over the DP state crashes it. */ - if (tphy->dp_up && !tphy->dp_mode_two_lanes && !t6040_dp_only(tphy)) { - t6040_phy_tx_cmn_down(tphy); - t6040_usb3_sleep(tphy); + if (tphy->dp_up && !tphy->dp_mode_two_lanes) { + t6040_dp_shutdown(tphy); dev_info(tphy->dev, "DP PHY down, PHY asleep\n"); } tphy->dp_up = false; @@ -1949,339 +2075,1077 @@ static const struct phy_ops t6040_dp_phy_ops = { .set_mode = t6040_dp_set_mode, }; -/* ---- Type-C mode switch: the Type-C controller asks for DP on a USB-C port ---- */ - -static int t6040_dp_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) -{ - struct t6040_usb2_phy *tphy = typec_mux_get_drvdata(mux); - bool dp = state->alt && state->alt->svid == USB_TYPEC_DP_SID; - bool alt = false; +/* ---- DP tunnels: the pixel clocks of the router's DP IN adapters (PHY in CIO) ---- + * + * The DP 1.4 tunnel's init, update and shutdown and the DP clock + * configuration: one fixed AUSPLL frequency feeds the DPTX pixel clocks, + * whose select field divides it per link rate. DP IN adapter 0 is taken to + * use PCLK1 and clock switch 1, adapter 1 PCLK2 and switch 2 (the PHY does + * not choose: the index is the display controller's). Driven only by + * appledrm through apple_t6040_dp_tunnel_rate(), and only while the PHY is in + * CIO. + */ +#define T6040_CORE_RCAL1 0x804 /* AUS40CMN_DIG RCAL1 */ +#define T6040_CORE_RCAL1_DONE BIT(0) +#define T6040_CORE_AUSPLL_CMD 0x3000 /* AUSPLL_TOP_APB_CMD_CMDOVERRIDE */ +#define T6040_CORE_AUSPLL_CMD_REQ BIT(0) +#define T6040_CORE_AUSPLL_CMD_ACK BIT(1) +#define T6040_CORE_AUSPLL_FREQDESC_A 0x3080 +#define T6040_CORE_AUSPLL_FREQDESC_B 0x3084 +#define T6040_CORE_AUSPLL_FREQDESC_C 0x3088 +#define T6040_CORE_AUSPLL_CLKOUT_MASTER 0x3200 +#define T6040_CORE_AUSPLL_CLKOUT_DIV 0x3208 +#define T6040_CORE_AUSPLL_BGR 0x3218 +#define T6040_CORE_AUSPLL_FREQINIT_1 0x3234 +#define T6040_CORE_TX_DP_CTRL0 0x7000 +#define T6040_CORE_TX_DP_CTRL0_ENABLES GENMASK(18, 16) +#define T6040_CORE_DP_PCLK_STATUS 0x7034 +#define T6040_CORE_DP_PCLK_STATUS_LOCK BIT(3) /* AUSPLL_LOCK */ + +static const struct t6040_tun_pclk { + u8 sel_shift; /* TX_DP_CTRL0 select field */ + u32 en; /* TX_DP_CTRL0 enable */ + u32 sw_mask, sw_gate, sw_on; /* clock switch: cleared, set, status when on */ + u32 req[2], req_stat[2], sw_off; /* PMGR requests, their status, cleared at off */ +} t6040_tun_pclk[2] = { + { 4, BIT(16), 0x473, 0x10, BIT(0), { 0x20, 0x40 }, { BIT(1), BIT(2) }, 0x403 }, + { 8, BIT(17), 0xb8c, 0x80, BIT(3), { 0x100, 0x200 }, { BIT(4), BIT(5) }, 0x80c }, +}; - guard(mutex)(&tphy->lock); - if (t6040_cio_state(state)) { - if (tphy->cio_up) - return 0; - return t6040_cio_up_locked(tphy); - } - t6040_cio_down_locked(tphy); - tphy->dp_flip = (tphy->orientation == TYPEC_ORIENTATION_REVERSE); - /* - * Latch the DP alt mode state for the DCP's Type-C route. Pin - * assignments C and E put DP on all four lanes. D leaves the USB3 - * pair with its running coprocessor and gives DP the other pair. - * The DCP brings the DP lanes up through phy_power_on()/phy_configure(). - */ - if (dp) { - switch (state->mode) { - case TYPEC_DP_STATE_C: - case TYPEC_DP_STATE_E: - alt = true; - break; - case TYPEC_DP_STATE_D: - alt = true; - break; - default: - break; - } +/* The pixel clock select: RBR, HBR, HBR2, HBR3; UHBR is not tunnelled on this path. */ +static int t6040_tun_select(u8 rate) +{ + switch (rate) { + case 0x06: + return 4; + case 0x0a: + return 3; + case 0x14: + return 1; + case 0x1e: + return 0; + default: + return -EINVAL; } - tphy->dp_alt = alt; - tphy->dp_two_lanes = alt && state->mode == TYPEC_DP_STATE_D; - dev_dbg(tphy->dev, "Type-C mux: %s state %lu -> DP alt mode %d, lane map %s\n", - dp ? "DP" : "non-DP", state->mode, alt, tphy->dp_flip ? "reversed" : "normal"); - if (alt && tphy->dp_up && !tphy->dp_mode_two_lanes && !tphy->dp_two_lanes) - t6040_dp_xbar_config(tphy, tphy->dp_flip); - t6040_usb3_reclaim_locked(tphy); - return 0; } -static void t6040_dp_mux_unregister(void *mux) +/* DP IN 1 runs on PCLK2 and needs PCLK1 as well: without it its video stops. */ +static u32 t6040_tun_enables(struct t6040_usb2_phy *tphy) { - typec_mux_unregister(mux); + return ((tphy->tun_rate[0] || tphy->tun_rate[1]) ? t6040_tun_pclk[0].en : 0) | + (tphy->tun_rate[1] ? t6040_tun_pclk[1].en : 0); } -/* - * Everything the DP route needs from the device tree; a node without it gets - * no DP phy. A USB-C PHY offers DP through a Type-C mode switch; a fixed DP - * route (an HDMI converter's PHY) is driven straight by the DCP with no mux - * and uses the normal lane map. - */ -static int t6040_dp_probe(struct t6040_usb2_phy *tphy) +/* AUSPLL_TOP command override: request @req with the toggle protocol, then wait for @ack. */ +static int t6040_tun_pll_cmd(struct t6040_usb2_phy *tphy, u32 val, bool ack_set) { - struct device *dev = tphy->dev; - unsigned int i; - - tphy->dp_rate = -1; + void __iomem *cmd = t6040_core(tphy) + T6040_CORE_AUSPLL_CMD; + u32 v; - /* - * DP mode needs the "core" and "cfg" windows. A USB-C PHY has them - * alongside its USB regions; a fixed DP PHY has them without any - * pipehandler or USB3 coprocessor. A node with neither is USB2 only. - */ - if (!tphy->regs[T6040_USB3_CORE] || !tphy->regs[T6040_USB3_CFG]) - return 0; + writel((readl(cmd) & 0xe0000006) | val, cmd); + return readl_poll_timeout(cmd, v, !!(v & T6040_CORE_AUSPLL_CMD_ACK) == ack_set, + 10, 500 * USEC_PER_MSEC); +} - tphy->dp = devm_phy_create(dev, NULL, &t6040_dp_phy_ops); - if (IS_ERR(tphy->dp)) - return PTR_ERR(tphy->dp); - phy_set_drvdata(tphy->dp, tphy); +/* The DP 1.4 tunnel shutdown: pixel clocks off, AUSPLL released. */ +static void t6040_tun_pll_stop(struct t6040_usb2_phy *tphy) +{ + void __iomem *core = t6040_core(tphy); + bool ack = readl(core + T6040_CORE_AUSPLL_CMD) & T6040_CORE_AUSPLL_CMD_ACK; + u32 v; - /* DP-only node: the generic apply_tunables tables (absent = not applied). */ - if (t6040_dp_only(tphy)) { - for (i = 0; i < T6040_DP_NR_GENERIC_TUNABLES; i++) { - const char *prop = t6040_usb3_tunable_list[i].prop; + t6040_usb2_clear32(core + T6040_CORE_TX_DP_CTRL0, T6040_CORE_TX_DP_CTRL0_ENABLES); + t6040_usb2_clear32(core + T6040_CORE_TX_DP_CTRL0, 0xc); + if (t6040_tun_pll_cmd(tphy, (ack ? 0 : T6040_CORE_AUSPLL_CMD_REQ) | 0x10000000, !ack) || + t6040_tun_pll_cmd(tphy, (ack ? T6040_CORE_AUSPLL_CMD_REQ : 0) | 0x10000018, ack) || + readl_poll_timeout(core + T6040_CORE_DP_PCLK_STATUS, v, + !(v & T6040_CORE_DP_PCLK_STATUS_LOCK), 10, 500 * USEC_PER_MSEC)) + dev_warn(tphy->dev, "DP tunnel: AUSPLL release timed out (cmd %#x, pclk status %#x)\n", + readl(core + T6040_CORE_AUSPLL_CMD), + readl(core + T6040_CORE_DP_PCLK_STATUS)); + t6040_usb2_clear32(core + T6040_CORE_AUSPLL_CLKOUT_MASTER, 0x8); + tphy->tun_pll = false; +} - if (!of_property_present(dev->of_node, prop) || - !tphy->res[t6040_usb3_tunable_list[i].region]) { - dev_warn(dev, "DP: no %s\n", prop); - tphy->tunables[i] = NULL; - continue; - } - struct resource *res = tphy->res[t6040_usb3_tunable_list[i].region]; +/* The common block's init, then the fixed AUSPLL program of the DP 1.4 tunnel init. */ +static int t6040_tun_pll_start(struct t6040_usb2_phy *tphy) +{ + void __iomem *core = t6040_core(tphy); + bool ack; + u32 v; - tphy->tunables[i] = devm_apple_tunable_parse(dev, dev->of_node, prop, res); - if (IS_ERR(tphy->tunables[i])) { - dev_warn(dev, "DP: %s: %pe\n", prop, tphy->tunables[i]); - tphy->tunables[i] = NULL; - } - } + if ((readl(core + T6040_CORE_AUSPLL_CLKOUT_MASTER) & 0x808) || + (readl(core + T6040_CORE_AUSPLL_CMD) & T6040_CORE_AUSPLL_CMD_REQ)) { + dev_warn(tphy->dev, "DP tunnel: AUSPLL already in use (master %#x, cmd %#x)\n", + readl(core + T6040_CORE_AUSPLL_CLKOUT_MASTER), + readl(core + T6040_CORE_AUSPLL_CMD)); + return -EBUSY; } + t6040_usb2_clear32(core + T6040_CORE_TX_DP_CTRL0, T6040_CORE_TX_DP_CTRL0_ENABLES); - if (of_property_present(dev->of_node, "mode-switch")) { - struct typec_mux_desc mux_desc = { - .drvdata = tphy, - .fwnode = dev_fwnode(dev), - .set = t6040_dp_mux_set, - }; - - tphy->mux = typec_mux_register(dev, &mux_desc); - if (IS_ERR(tphy->mux)) - return PTR_ERR(tphy->mux); - if (devm_add_action_or_reset(dev, t6040_dp_mux_unregister, tphy->mux)) - return -ENOMEM; + t6040_usb2_set32(core + T6040_CORE_CMN_OVRD, 0x4); + t6040_usb2_set32(core + T6040_CORE_CMN_OVRD, 0x8); + udelay(1); + t6040_usb2_set32(core + T6040_CORE_CMN_OVRD, 0x1); + t6040_usb2_set32(core + T6040_CORE_CMN_OVRD, 0x2); + udelay(1); + t6040_usb2_clear32(core + T6040_CORE_CMN_OVRD, T6040_CORE_CMN_OVRD_CLAMP); + t6040_usb2_set32(core + T6040_CORE_CMN_OVRD, 0x20); + udelay(1); + t6040_usb2_set32(core + T6040_CORE_CMN_BIAS, T6040_CORE_CMN_BIAS_BGBIAS_EN_OV); + udelay(15); + if (readl_poll_timeout(core + T6040_CORE_RCAL1, v, v & T6040_CORE_RCAL1_DONE, 1, 100)) + dev_dbg(tphy->dev, "DP tunnel: RCAL not done (%#x)\n", v); + + t6040_usb2_clear32(core + T6040_CORE_AUSPLL_FREQINIT_1, 0x3); + writel(0x1e0e021c, core + T6040_CORE_AUSPLL_FREQDESC_A); + t6040_usb2_mask32(core + T6040_CORE_AUSPLL_FREQDESC_B, 0x0fffffff, 0); + t6040_usb2_mask32(core + T6040_CORE_AUSPLL_FREQDESC_C, 0x007fffff, 0x644a00); + t6040_usb2_mask32(core + T6040_CORE_AUSPLL_CLKOUT_DIV, 0x001f0000, 0x10000); + t6040_usb2_set32(core + T6040_CORE_AUSPLL_BGR, 0x1); + t6040_usb2_set32(core + T6040_CORE_AUSPLL_CLKOUT_MASTER, 0x808); + tphy->tun_pll = true; + + ack = readl(core + T6040_CORE_AUSPLL_CMD) & T6040_CORE_AUSPLL_CMD_ACK; + if (t6040_tun_pll_cmd(tphy, (ack ? 0 : T6040_CORE_AUSPLL_CMD_REQ) | 0x10000000, true) || + readl_poll_timeout(core + T6040_CORE_DP_PCLK_STATUS, v, + !!(v & T6040_CORE_DP_PCLK_STATUS_LOCK) == !ack, + 10, 500 * USEC_PER_MSEC) || + t6040_tun_pll_cmd(tphy, (ack ? T6040_CORE_AUSPLL_CMD_REQ : 0) | 0x10010000, ack)) { + dev_err(tphy->dev, "DP tunnel: AUSPLL did not lock (cmd %#x, pclk status %#x)\n", + readl(core + T6040_CORE_AUSPLL_CMD), + readl(core + T6040_CORE_DP_PCLK_STATUS)); + t6040_tun_pll_stop(tphy); + return -ETIMEDOUT; } - - tphy->dp_ok = true; + t6040_usb2_set32(core + T6040_CORE_TX_DP_CTRL0, 0xc); return 0; } -/* ---- the coprocessor's RTKit ---- */ +static void t6040_tun_select_field(u32 *v, const struct t6040_tun_pclk *p, int sel, bool en) +{ + if (!!(*v & p->en) != en || ((*v >> p->sel_shift) & 0xf) != sel) + *v = (*v & ~(p->en | (0xf << p->sel_shift))) | (sel << p->sel_shift); +} -/* The coprocessor places its buffers in its own __DATA SRAM. */ -static int t6040_cpu_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +/* The DP 1.4 tunnel update for one pclk: select with its enable off, then the enables. */ +static void t6040_tun_select_write(struct t6040_usb2_phy *tphy, unsigned int dpin, int sel) { - struct t6040_usb2_phy *tphy = cookie; - resource_size_t size = resource_size(tphy->cpu_data); + void __iomem *ctrl0 = t6040_core(tphy) + T6040_CORE_TX_DP_CTRL0; + u32 v = readl(ctrl0); - if (!bfr->iova || bfr->iova < T6040_CPU_DATA_IOVA || bfr->size > size || - bfr->iova - T6040_CPU_DATA_IOVA > size - bfr->size) { - dev_err(tphy->dev, "USB3: coprocessor buffer %pad+%#zx outside its SRAM\n", - &bfr->iova, bfr->size); - return -EINVAL; - } - bfr->iomem = ioremap_np(tphy->cpu_data->start + (bfr->iova - T6040_CPU_DATA_IOVA), - bfr->size); - if (!bfr->iomem) - return -ENOMEM; - bfr->is_mapped = true; - return 0; + t6040_tun_select_field(&v, &t6040_tun_pclk[dpin], sel, !!tphy->tun_rate[dpin]); + /* PCLK1 is DP IN 1's prerequisite at its own rate, unless DP IN 0 runs on it. */ + if (dpin == 1 && !tphy->tun_rate[0]) + t6040_tun_select_field(&v, &t6040_tun_pclk[0], sel, true); + writel(v, ctrl0); + writel((v & ~T6040_CORE_TX_DP_CTRL0_ENABLES) | t6040_tun_enables(tphy), ctrl0); } -static void t6040_cpu_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +/* The DP clock configuration for one DP 1.x switch: pixel-clock gate on, or off. */ +static int t6040_tun_switch(struct t6040_usb2_phy *tphy, unsigned int dpin, bool on) { - if (bfr->iomem) - iounmap(bfr->iomem); + const struct t6040_tun_pclk *p = &t6040_tun_pclk[dpin]; + void __iomem *cfg = t6040_cfg(tphy); + unsigned int i; + u32 s; + + if (on) { + t6040_usb2_mask32(cfg + T6040_CFG_DP_CLKSWITCH_CTRL, p->sw_mask, p->sw_gate); + return readl_poll_timeout(cfg + T6040_CFG_DP_CLKSWITCH_STAT, s, s & p->sw_on, + 10, 5 * USEC_PER_MSEC); + } + t6040_usb2_set32(cfg + T6040_CFG_DP_CLKSWITCH_CTRL, p->sw_gate); + for (i = 0; i < 2; i++) { + if (!(readl(cfg + T6040_CFG_DP_CLKSWITCH_CTRL) & p->req[i])) + continue; + t6040_usb2_clear32(cfg + T6040_CFG_DP_CLKSWITCH_CTRL, p->req[i]); + if (readl_poll_timeout(cfg + T6040_CFG_DP_CLKSWITCH_STAT, s, !(s & p->req_stat[i]), + 10, 5 * USEC_PER_MSEC)) + dev_dbg(tphy->dev, "DP tunnel %u: request %#x not released (%#x)\n", + dpin, p->req[i], s); + } + t6040_usb2_clear32(cfg + T6040_CFG_DP_CLKSWITCH_CTRL, p->sw_off); + return 0; } -static void t6040_cpu_crashed(void *cookie, const void *crashlog, size_t size) +/* All tunnel clocks down, which the PHY's shutdown of mode 2 starts with. */ +static void t6040_tun_stop_all_locked(struct t6040_usb2_phy *tphy) { - struct t6040_usb2_phy *tphy = cookie; + unsigned int dpin; - dev_err(tphy->dev, "USB3: the PHY coprocessor crashed\n"); + lockdep_assert_held(&tphy->lock); + if (!tphy->tun_pll) + return; + for (dpin = 0; dpin < 2; dpin++) { + if (!tphy->tun_rate[dpin]) + continue; + tphy->tun_rate[dpin] = 0; + t6040_tun_switch(tphy, dpin, false); + } + if (tphy->tun_pre) { + tphy->tun_pre = false; + t6040_tun_switch(tphy, 0, false); + } + t6040_tun_pll_stop(tphy); + dev_info(tphy->dev, "DP tunnel clocks off\n"); } -static void t6040_cpu_recv_message(void *cookie, u8 ep, u64 msg) +static int t6040_tun_rate_locked(struct t6040_usb2_phy *tphy, unsigned int dpin, u8 rate) { - struct t6040_usb2_phy *tphy = cookie; + int sel, ret; + u8 old = tphy->tun_rate[dpin]; - if (ep == T6040_CPU_EP_APP) { - if (atomic_xchg(&tphy->drop_next_power_down_ack, 0)) { - dev_warn(tphy->dev, "USB3: dropped Power Down acknowledge (fault injection)\n"); - return; + lockdep_assert_held(&tphy->lock); + if (!rate) { + if (!old) + return 0; + tphy->tun_rate[dpin] = 0; + if (dpin == 0 && tphy->tun_rate[1]) { + /* DP IN 1 keeps PCLK1 and its switch, now at its own rate. */ + tphy->tun_pre = true; + t6040_tun_select_write(tphy, 1, t6040_tun_select(tphy->tun_rate[1])); + } else { + t6040_tun_switch(tphy, dpin, false); + if (dpin == 1 && tphy->tun_pre) { + tphy->tun_pre = false; + t6040_tun_switch(tphy, 0, false); + } } - dev_dbg(tphy->dev, "USB3: Power Down acknowledge %#llx\n", msg); - complete(&tphy->app_reply); - } else { - dev_dbg(tphy->dev, "USB3: coprocessor message on endpoint %u: %#llx\n", ep, msg); + t6040_usb2_mask32(t6040_core(tphy) + T6040_CORE_TX_DP_CTRL0, + T6040_CORE_TX_DP_CTRL0_ENABLES, t6040_tun_enables(tphy)); + if (!tphy->tun_rate[!dpin]) + t6040_tun_pll_stop(tphy); + dev_dbg(tphy->dev, "DP tunnel %u off\n", dpin); + return 0; + } + sel = t6040_tun_select(rate); + if (sel < 0) + return sel; + if (!tphy->cio_up) + return -EBUSY; + if (old == rate) + return 0; + if (!tphy->tun_pll) { + ret = t6040_tun_pll_start(tphy); + if (ret) + return ret; + } + tphy->tun_rate[dpin] = rate; + t6040_tun_select_write(tphy, dpin, sel); + /* DP IN 1 alone: switch 1 passes PCLK1, its prerequisite. */ + if (dpin == 1 && !tphy->tun_rate[0] && !tphy->tun_pre) { + ret = t6040_tun_switch(tphy, 0, true); + if (ret) + goto switch_failed; + tphy->tun_pre = true; + } else if (dpin == 0) { + tphy->tun_pre = false; /* DP IN 0 owns switch 1 again */ + } + ret = t6040_tun_switch(tphy, dpin, true); +switch_failed: + if (ret) { + dev_err(tphy->dev, "DP tunnel %u: clock switch timed out (ctrl %#x status %#x)\n", + dpin, readl(t6040_cfg(tphy) + T6040_CFG_DP_CLKSWITCH_CTRL), + readl(t6040_cfg(tphy) + T6040_CFG_DP_CLKSWITCH_STAT)); + t6040_tun_rate_locked(tphy, dpin, 0); + return ret; } + dev_info(tphy->dev, "DP tunnel %u: rate %#x, select %d (ctrl0 %#x, pclk status %#x)\n", + dpin, rate, sel, readl(t6040_core(tphy) + T6040_CORE_TX_DP_CTRL0), + readl(t6040_core(tphy) + T6040_CORE_DP_PCLK_STATUS)); + return 0; } -static const struct apple_rtkit_ops t6040_cpu_rtkit_ops = { - .flags = APPLE_RTKIT_COPROC_PLACES_BUFFERS, - .crashed = t6040_cpu_crashed, - .recv_message = t6040_cpu_recv_message, - .shmem_setup = t6040_cpu_shmem_setup, - .shmem_destroy = t6040_cpu_shmem_destroy, +/* + * The pixel clock of a DP tunnel for appledrm, which looks it up with symbol_get(): start the + * clock of DP IN adapter @dpin (0 or 1) at DP link rate code @rate, or stop it with 0. The + * PHY must be in CIO. A symbol of its own, as for T6050: phy-apple-atc exports + * apple_atc_dp_tunnel_rate() for the older SoCs, and both can be built into one kernel. + */ +int apple_t6040_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) +{ + struct t6040_usb2_phy *tphy; + + if (!phy || phy->ops != &t6040_dp_phy_ops || dpin > 1) + return -EINVAL; + tphy = phy_get_drvdata(phy); + guard(mutex)(&tphy->lock); + return t6040_tun_rate_locked(tphy, dpin, rate); +} +EXPORT_SYMBOL_GPL(apple_t6040_dp_tunnel_rate); + +/* ---- CIO: USB4/Thunderbolt (PHY mode 2) ---- + * + * The PHY init for mode 2 with the orientation: the power-up reset (shared + * with USB3), the generic ADT tables, the CIO core tunables (the same + * three core tables as USB3) and the CIO-only tunables (the CIO lane + * tables, applied in every CIO session: nothing on the PHY side picks 40G or + * 80G), the mode-2 crossbar, the mode-2 sleep-control init (common and both + * lanes, receivers powered by override), then part C: the PHY coprocessor + * firmware tunes the PHY as for USB3, without the USB lane-swap bit. The + * firmware's only message, Power Down, is sent at exit as for + * USB3. The Type-C controller sets the mux to USB4/TBT after the orientation + * and before it hands the cable to the host router (drivers/thunderbolt/apple.c), + * and back to SAFE after the router has stopped; the PHY holds a runtime PM + * reference in between. CIO is offered only with the CIO tables in the DT; + * without them every mux state takes the path it took before. + * + * USB3 runs tunnelled through the router: the controller's PIPE goes to the + * USB4 backend (see t6040_cio_pipe_tunnel()) while the PHY is in CIO and the + * router has asked for it. + */ + +/* The ADT tables of the CIO-only tunables, in order; lane0/1 are required. */ +static const char * const t6040_cio_tunable_props[] = { + "apple,tunable-cio-uc-regs", /* tunable_UC_REGS_CIO_DFLT, core + 0x400 */ + "apple,tunable-cio-lane0", /* tunable_LN0_RX_TOP_CIO_DFLT, core + 0x9000 */ + "apple,tunable-cio-lane1", /* tunable_LN1_RX_TOP_CIO_DFLT, core + 0x19000 */ + "apple,tunable-cio-lane0-eq", /* tunable_LN0_RX_EQ_CIO_DFLT, core + 0xb000 */ + "apple,tunable-cio-lane1-eq", /* tunable_LN1_RX_EQ_CIO_DFLT, core + 0x1b000 */ }; -static void t6040_rtkit_cleanup(void *data) +static_assert(ARRAY_SIZE(t6040_cio_tunable_props) == + ARRAY_SIZE(((struct t6040_usb2_phy *)NULL)->cio_tunables)); + +/* The PIPE backend of USB3 tunnelled through the USB4 router */ +#define T6040_PIPEHANDLER_MUX_CLK_USB4 2 +#define T6040_PIPEHANDLER_MUX_DATA_USB4 1 + +/* The tunables of mode 2: generic tables, the CIO core tunables, the CIO-only tunables. */ +static void t6040_cio_tunables(struct t6040_usb2_phy *tphy) { - t6040_rtkit_stop_transport(data); + size_t i; + + for (i = 0; i < T6040_USB3_NR_TUNABLES; i++) + apple_tunable_apply(tphy->regs[t6040_usb3_tunable_list[i].region], + tphy->tunables[i]); + for (i = 0; i < ARRAY_SIZE(tphy->cio_tunables); i++) + apple_tunable_apply(t6040_core(tphy), tphy->cio_tunables[i]); } -static void t6040_mbox_dev_put(void *data) +/* The mode-2 crossbar: lane fields of core+0x6c cleared, core+0x78[4:0] = flip, then [1:0] = 2. */ +static void t6040_cio_xbar_config(struct t6040_usb2_phy *tphy, bool flip) { - put_device(data); + void __iomem *core = t6040_core(tphy); + + t6040_usb2_mask32(core + T6040_CORE_XBAR, ~0xffffff03u, 0); + t6040_usb2_mask32(core + T6040_CORE_XBAR_LANES, 0x1f, flip ? 1 : 0); + t6040_usb2_mask32(core + T6040_CORE_XBAR, 0x3, 0x2); } -static int t6040_get_mbox_dev(struct t6040_usb2_phy *tphy) +struct t6040_cio_step { + u16 off; + u32 bit; /* lane 0; lane 1 is the next bit up */ + bool set; +}; + +static void t6040_cio_steps(struct t6040_usb2_phy *tphy, const struct t6040_cio_step *step, + size_t n, unsigned int lane) { - struct of_phandle_args args; - struct platform_device *pdev; - int ret; + size_t i; - ret = of_parse_phandle_with_args(tphy->dev->of_node, "mboxes", "#mbox-cells", 0, - &args); - if (ret) - return ret; - pdev = of_find_device_by_node(args.np); - of_node_put(args.np); - if (!pdev) - return -EPROBE_DEFER; + for (i = 0; i < n; i++) { + t6040_usb2_mask32(t6040_core(tphy) + step[i].off, step[i].bit << lane, + step[i].set ? step[i].bit << lane : 0); + udelay(1); + } +} - tphy->mbox_dev = &pdev->dev; - return devm_add_action_or_reset(tphy->dev, t6040_mbox_dev_put, tphy->mbox_dev); +/* RX power-up, per lane: the lane's receiver powered by override (CFG1_TOP, CFG0_TOP). */ +static const struct t6040_cio_step t6040_cio_pwr_up_rxa[] = { + { T6040_CORE_CFG1_TOP, 0x10, true }, { T6040_CORE_CFG1_TOP, 0x40, true }, + { T6040_CORE_CFG1_TOP, 0x1, true }, { T6040_CORE_CFG1_TOP, 0x4, true }, + { T6040_CORE_CMN_OVRD, 0x400, true }, { T6040_CORE_CMN_OVRD, 0x1000, true }, + { T6040_CORE_CMN_OVRD, 0x40, true }, { T6040_CORE_CMN_OVRD, 0x100, true }, + { T6040_CORE_CMN_OVRD, 0x4000, false }, { T6040_CORE_CMN_OVRD, 0x10000, true }, +}; + +/* TX override removal, per lane: the transmitter's overrides released (CFG_SLP_CTRL). */ +static const struct t6040_cio_step t6040_cio_tx_rm_ovrd[] = { + { T6040_CORE_SLP_CTRL, 0x40, false }, { T6040_CORE_SLP_CTRL, 0x4, false }, + { T6040_CORE_SLP_CTRL, 0x400, false }, +}; + +/* RX power-down, per lane */ +static const struct t6040_cio_step t6040_cio_rx_powerdown[] = { + { T6040_CORE_CMN_OVRD, 0x4000, true }, { T6040_CORE_CMN_OVRD, 0x40, false }, + { T6040_CORE_CMN_OVRD, 0x400, false }, { T6040_CORE_CFG1_TOP, 0x1, false }, + { T6040_CORE_CFG1_TOP, 0x10, false }, +}; + +/* + * The mode-2 sleep-control init: the common block's overrides removed, then + * the RX power-up and the TX override removal per lane. + */ +static void t6040_cio_slpctrl(struct t6040_usb2_phy *tphy) +{ + unsigned int lane; + + t6040_usb2_clear32(t6040_core(tphy) + T6040_CORE_CMN_OVRD, 0x2a); + udelay(1); + for (lane = 0; lane < 2; lane++) { + t6040_cio_steps(tphy, t6040_cio_pwr_up_rxa, ARRAY_SIZE(t6040_cio_pwr_up_rxa), lane); + t6040_cio_steps(tphy, t6040_cio_tx_rm_ovrd, ARRAY_SIZE(t6040_cio_tx_rm_ovrd), lane); + } } -static void t6040_debugfs_remove(void *data) +/* + * The PIPE from the dummy backend to the USB4 router, with the lock handshake + * only while the controller is out of reset. No non-selected override and no + * BIST bracket, unlike the PHY's own USB3. + */ +static void t6040_cio_pipe_tunnel(struct t6040_usb2_phy *tphy) { - debugfs_remove_recursive(data); + void __iomem *ph = tphy->pipehandler; + bool running = readl(ph + T6040_PIPEHANDLER_AON_GEN) & + T6040_PIPEHANDLER_AON_GEN_DWC3_RESET_N; + + if (readl(ph + T6040_PIPEHANDLER_LOCK_ACK) & T6040_PIPEHANDLER_LOCK_EN) + dev_dbg(tphy->dev, "CIO: PIPE lock already acknowledged\n"); + t6040_usb2_clear32(ph + T6040_PIPEHANDLER_OVERRIDE_VALUES, 0xe); + t6040_usb2_set32(ph + T6040_PIPEHANDLER_OVERRIDE, T6040_PIPEHANDLER_OVERRIDE_EN); + if (running && t6040_usb3_pipe_lock(tphy, 6)) + dev_dbg(tphy->dev, "CIO: PIPE lock not acknowledged, switching anyway\n"); + t6040_usb3_pipe_mux(tphy, T6040_PIPEHANDLER_MUX_CLK_USB4, + T6040_PIPEHANDLER_MUX_DATA_USB4); + t6040_usb2_clear32(ph + T6040_PIPEHANDLER_OVERRIDE, T6040_PIPEHANDLER_OVERRIDE_EN); + t6040_usb2_clear32(ph + T6040_PIPEHANDLER_OVERRIDE, T6040_PIPEHANDLER_OVERRIDE_EN2); + if (running) + t6040_usb3_pipe_unlock(tphy); + tphy->pipe_tunneled = true; + dev_info(tphy->dev, "USB3 PIPE on the USB4 router\n"); } -static int t6040_debugfs_init(struct t6040_usb2_phy *tphy) +/* + * The PIPE is on the router while the router runs and carries USB3 for the + * controller (only a tunnelled USB3 transport needs it there, and only once + * it uses the tunnel), dwc3 holds the USB3 phy with its PIPE and the + * controller is out of reset. Otherwise it goes back to the dummy. + */ +static void t6040_cio_pipe_update_locked(struct t6040_usb2_phy *tphy) { - const char *name; - int ret; + bool want; - name = devm_kasprintf(tphy->dev, GFP_KERNEL, "t6040-atcphy-%s", - dev_name(tphy->dev)); - if (!name) - return -ENOMEM; - tphy->debugfs = debugfs_create_dir(name, NULL); - if (IS_ERR(tphy->debugfs)) { - if (PTR_ERR(tphy->debugfs) == -ENODEV) { - tphy->debugfs = NULL; - return 0; - } - return PTR_ERR(tphy->debugfs); + lockdep_assert_held(&tphy->lock); + want = tphy->cio_up && tphy->router_running && tphy->usb3_tunnel_req && + tphy->usb3_on && tphy->usb3_pipe && tphy->dwc3_running && !tphy->pipe_switched; + if (want && !tphy->pipe_tunneled) { + t6040_cio_pipe_tunnel(tphy); + } else if (!want && tphy->pipe_tunneled) { + t6040_usb3_pipe_park(tphy); + tphy->pipe_tunneled = false; + dev_info(tphy->dev, "USB3 PIPE back on the dummy PHY\n"); } - debugfs_create_atomic_t("drop_next_power_down_ack", 0600, tphy->debugfs, - &tphy->drop_next_power_down_ack); - ret = devm_add_action_or_reset(tphy->dev, t6040_debugfs_remove, tphy->debugfs); - if (ret) - tphy->debugfs = NULL; - return ret; } /* - * Map the PHY register windows shared by the USB3 and DisplayPort paths. Each - * is optional: a USB-C PHY has the full set, the fixed HDMI DP PHY (atcphy3) - * lacks none of core/cfg but has no pipehandler. Returns true once at least - * "core" and "cfg" are mapped, which is all DP mode needs. + * The PHY shutdown of mode 2 once the PIPE is off the router: part A (both TX + * lanes and the common block, then the receivers), the coprocessor down, part + * B. Also what a bring-up that failed after the lanes' power-up has to undo. */ -static bool t6040_map_regions(struct t6040_usb2_phy *tphy, struct platform_device *pdev) +static void t6040_cio_phy_shutdown(struct t6040_usb2_phy *tphy) { - struct device *dev = tphy->dev; - size_t i; + unsigned int lane; - for (i = 0; i < T6040_USB3_NR_REGIONS; i++) { - tphy->res[i] = platform_get_resource_byname(pdev, IORESOURCE_MEM, - t6040_usb3_region_names[i]); - if (!tphy->res[i]) - continue; - tphy->regs[i] = devm_ioremap_resource(dev, tphy->res[i]); - if (IS_ERR(tphy->regs[i])) { - dev_warn(dev, "\"%s\" region: %pe\n", - t6040_usb3_region_names[i], tphy->regs[i]); - tphy->regs[i] = NULL; - tphy->res[i] = NULL; - } + t6040_phy_tx_cmn_shutdown(tphy); + for (lane = 0; lane < 2; lane++) + t6040_cio_steps(tphy, t6040_cio_rx_powerdown, ARRAY_SIZE(t6040_cio_rx_powerdown), + lane); + fsleep(200); + t6040_usb3_firmware_power_down(tphy); + t6040_usb3_sleep(tphy); +} + +/* The PHY out of mode 2: the tunnel clocks, the PIPE, then the PHY shutdown. */ +static void t6040_cio_down_locked(struct t6040_usb2_phy *tphy) +{ + u32 uc; + + lockdep_assert_held(&tphy->lock); + if (!tphy->cio_up) + return; + t6040_tun_stop_all_locked(tphy); + /* Off the router: the same writes as the park from this PHY */ + if (tphy->pipe_tunneled) { + t6040_usb3_pipe_park(tphy); + tphy->pipe_tunneled = false; } - return tphy->regs[T6040_USB3_CORE] && tphy->regs[T6040_USB3_CFG]; + uc = readl(t6040_core(tphy) + T6040_CORE_UC_CTRL); + t6040_cio_phy_shutdown(tphy); + dev_info(tphy->dev, "CIO PHY asleep: uc %#x -> %#x, cfg %#x/%#x\n", uc, + readl(t6040_core(tphy) + T6040_CORE_UC_CTRL), + readl(t6040_cfg(tphy) + T6040_CFG_CTRL), + readl(t6040_cfg(tphy) + T6040_CFG_STATUS)); + tphy->cio_up = false; + pm_runtime_put(tphy->dev); } /* - * Everything USB3 needs from the device tree. A missing piece leaves the - * port at USB2; only a mailbox that has not probed yet defers. + * Bring the PHY into CIO for the plug orientation. A failure is returned to + * the Type-C controller, which then does not hand the cable to the router. */ -static int t6040_usb3_probe(struct t6040_usb2_phy *tphy, struct platform_device *pdev) +static int t6040_cio_up_locked(struct t6040_usb2_phy *tphy) { - struct device *dev = tphy->dev; - struct typec_switch_desc sw_desc = { - .drvdata = tphy, - .fwnode = dev_fwnode(dev), - .set = t6040_usb3_sw_set, - }; - struct typec_switch_dev *sw; - struct resource *ph; - size_t i; + void __iomem *core = t6040_core(tphy); + bool cycle; + bool flip; int ret; - tphy->usb3 = devm_phy_create(dev, NULL, &t6040_usb3_phy_ops); - if (IS_ERR(tphy->usb3)) - return PTR_ERR(tphy->usb3); - phy_set_drvdata(tphy->usb3, tphy); - init_completion(&tphy->app_reply); - - if (!of_property_present(dev->of_node, "orientation-switch")) + lockdep_assert_held(&tphy->lock); + if (tphy->cio_up) return 0; - /* - * Registered first: tipd waits for the switch the DT names, so the port - * keeps USB2, roles and PD even if a USB3 piece below is missing. - */ - sw = typec_switch_register(dev, &sw_desc); - if (IS_ERR(sw)) - return PTR_ERR(sw); - ret = devm_add_action_or_reset(dev, t6040_usb3_switch_unregister, sw); - if (ret) - return ret; - - for (i = 0; i < T6040_USB3_NR_REGIONS; i++) { - if (!tphy->regs[i]) { - dev_warn(dev, "USB3: no \"%s\" region, USB2 only\n", - t6040_usb3_region_names[i]); - return 0; - } - } - tphy->cpu_data = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpu-data"); - ph = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pipehandler"); - if (!tphy->cpu_data || !ph) { - dev_warn(dev, "USB3: no coprocessor SRAM region, USB2 only\n"); - return 0; + switch (tphy->orientation) { + case TYPEC_ORIENTATION_NORMAL: + flip = false; + break; + case TYPEC_ORIENTATION_REVERSE: + flip = true; + break; + default: + dev_warn(tphy->dev, "CIO: no plug orientation\n"); + return -ENODEV; } - - for (i = 0; i < T6040_USB3_NR_TUNABLES; i++) { - tphy->tunables[i] = - devm_apple_tunable_parse(dev, dev->of_node, t6040_usb3_tunable_list[i].prop, - tphy->res[t6040_usb3_tunable_list[i].region]); - if (IS_ERR(tphy->tunables[i])) { - dev_warn(dev, "USB3: %s: %pe, USB2 only\n", - t6040_usb3_tunable_list[i].prop, tphy->tunables[i]); - tphy->tunables[i] = NULL; - return 0; - } + if (tphy->dp_up) { + dev_warn(tphy->dev, "CIO: lanes held by DP\n"); + return -EBUSY; } - tphy->tunable_pipehandler = devm_apple_tunable_parse(dev, dev->of_node, - "apple,tunable-pipehandler", ph); - if (IS_ERR(tphy->tunable_pipehandler)) { - dev_warn(dev, "USB3: apple,tunable-pipehandler: %pe, USB2 only\n", - tphy->tunable_pipehandler); - return 0; + /* USB3 hands its lanes over and comes back once the port leaves CIO. */ + if (tphy->usb3_up || tphy->pipe_switched) { + t6040_usb3_down_locked(tphy); + tphy->usb3_lent = true; } - ret = t6040_get_mbox_dev(tphy); - if (ret) - return dev_err_probe(dev, ret, "failed to find PHY coprocessor mailbox\n"); + ret = pm_runtime_resume_and_get(tphy->dev); + if (ret) { + t6040_usb3_reclaim_locked(tphy); + return ret; + } + + /* The coprocessor's recovery, as in t6040_usb3_up_locked(): it is the same firmware. */ + cycle = t6040_recovery_start(&tphy->recovery); +retry: + if (cycle) { + ret = t6040_usb3_hard_recover(tphy); + if (ret) + goto put; + } + + ret = t6040_usb3_wake(tphy); + if (ret) + goto sleep; /* the coprocessor block may be unclocked: never touch it */ + t6040_cio_tunables(tphy); + t6040_cio_xbar_config(tphy, flip); + t6040_cio_slpctrl(tphy); + ret = t6040_phy_run_firmware(tphy, "CIO", flip); + if (ret) + goto retry_or_fail; + + t6040_recovery_complete(&tphy->recovery); + tphy->cio_up = true; + dev_info(tphy->dev, "CIO PHY up (mode 2), %s orientation: xbar %#x/%#x cfg %#x/%#x\n", + flip ? "reverse" : "normal", readl(core + T6040_CORE_XBAR), + readl(core + T6040_CORE_XBAR_LANES), readl(t6040_cfg(tphy) + T6040_CFG_CTRL), + readl(t6040_cfg(tphy) + T6040_CFG_STATUS)); + t6040_cio_pipe_update_locked(tphy); + return 0; + +retry_or_fail: + /* The lanes are powered by now: the full shutdown, and one fresh start may follow. */ + t6040_cio_phy_shutdown(tphy); + if (t6040_recovery_retry(&tphy->recovery)) { + cycle = true; + goto retry; + } + goto put; + +sleep: + t6040_usb3_sleep(tphy); +put: + pm_runtime_put(tphy->dev); + dev_warn(tphy->dev, "CIO PHY bring-up failed: %d\n", ret); + /* The port stays out of CIO: give the lanes back to USB3. */ + t6040_usb3_reclaim_locked(tphy); + return ret; +} + +/* + * A USB4/Thunderbolt mux event only takes the port into CIO while the router + * that runs over this PHY (a node whose phys property names it) is enabled + * in the live device tree. The default device trees keep the routers + * disabled, so the port stays on its USB2/USB3/DP paths until a router is + * enabled. + */ +static bool t6040_cio_router_enabled(struct t6040_usb2_phy *tphy) +{ + struct device_node *np; + + for_each_compatible_node(np, NULL, "apple,t6040-usb4-acio") { + struct device_node *phy = of_parse_phandle(np, "phys", 0); + bool match = phy == tphy->dev->of_node && of_device_is_available(np); + + of_node_put(phy); + if (match) { + of_node_put(np); + return true; + } + } + return false; +} + +/* USB4 entered with Enter_USB in USB4 mode, or Thunderbolt alt mode. */ +static bool t6040_cio_requested(const struct typec_mux_state *state) +{ + const struct enter_usb_data *data = state->data; + + if (state->alt) + return state->alt->svid == USB_TYPEC_TBT_SID; + return state->mode == TYPEC_MODE_USB4 && data && + FIELD_GET(EUDO_USB_MODE_MASK, data->eudo) == EUDO_USB_MODE_USB4; +} + +static void t6040_cio_remove(void *data) +{ + struct t6040_usb2_phy *tphy = data; + + guard(mutex)(&tphy->lock); + if (tphy->router_running) + dev_warn(tphy->dev, "CIO: unbound under a running USB4 router\n"); + tphy->router_running = false; + t6040_cio_down_locked(tphy); +} + +/* CIO needs the USB3 pieces and the CIO lane tables; anything less leaves it off. */ +static int t6040_cio_probe(struct t6040_usb2_phy *tphy) +{ + struct device *dev = tphy->dev; + size_t i; + + if (!tphy->usb3_ok || + (!of_property_present(dev->of_node, t6040_cio_tunable_props[1]) && + !of_property_present(dev->of_node, t6040_cio_tunable_props[2]))) + return 0; + + for (i = 0; i < ARRAY_SIZE(t6040_cio_tunable_props); i++) { + const char *prop = t6040_cio_tunable_props[i]; + struct apple_tunable *t; + + if (!of_property_present(dev->of_node, prop)) { + if (i == 1 || i == 2) { + dev_warn(dev, "CIO: no %s, no USB4/Thunderbolt\n", prop); + return 0; + } + continue; + } + t = devm_apple_tunable_parse(dev, dev->of_node, prop, tphy->res[T6040_USB3_CORE]); + if (IS_ERR(t)) { + dev_warn(dev, "CIO: %s: %pe, no USB4/Thunderbolt\n", prop, t); + return 0; + } + tphy->cio_tunables[i] = t; + } + + /* Registered before the mode switch, so it runs after the switch is gone. */ + if (devm_add_action_or_reset(dev, t6040_cio_remove, tphy)) + return -ENOMEM; + tphy->cio_ok = true; + dev_dbg(dev, "CIO (USB4/Thunderbolt) mode available\n"); + return 0; +} + +/* ---- Type-C mode switch: the Type-C controller asks for DP on a USB-C port ---- */ + +static int t6040_dp_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct t6040_usb2_phy *tphy = typec_mux_get_drvdata(mux); + bool dp = state->alt && state->alt->svid == USB_TYPEC_DP_SID; + bool alt = false; + + guard(mutex)(&tphy->lock); + /* USB4/TBT, only with the CIO tables; leaving it first takes the PHY out of CIO. */ + if (tphy->cio_ok && (tphy->cio_up || t6040_cio_router_enabled(tphy))) { + bool cio = t6040_cio_requested(state); + + if (tphy->cio_up && !cio) { + /* The router stops before the mux leaves USB4/TBT (apple.c). */ + if (tphy->router_running) { + dev_err(tphy->dev, "CIO: the USB4 router still runs, not leaving CIO\n"); + return -EBUSY; + } + t6040_cio_down_locked(tphy); + } + if (cio) { + tphy->dp_alt = false; + tphy->dp_two_lanes = false; + dev_dbg(tphy->dev, "Type-C mux: %s -> CIO\n", + state->alt ? "Thunderbolt" : "USB4"); + return t6040_cio_up_locked(tphy); + } + } + tphy->dp_flip = (tphy->orientation == TYPEC_ORIENTATION_REVERSE); + /* + * Latch the DP alt mode state for the DCP's Type-C route. Pin + * assignments C and E put DP on all four lanes. D leaves the USB3 + * pair with its running coprocessor and gives DP the other pair. + * The DCP brings the DP lanes up through phy_power_on()/phy_configure(). + */ + if (dp) { + switch (state->mode) { + case TYPEC_DP_STATE_C: + case TYPEC_DP_STATE_E: + alt = true; + break; + case TYPEC_DP_STATE_D: + alt = true; + break; + default: + break; + } + } + tphy->dp_alt = alt; + tphy->dp_two_lanes = alt && state->mode == TYPEC_DP_STATE_D; + dev_dbg(tphy->dev, "Type-C mux: %s state %lu -> DP alt mode %d, lane map %s\n", + dp ? "DP" : "non-DP", state->mode, alt, tphy->dp_flip ? "reversed" : "normal"); + if (alt && tphy->dp_up && !tphy->dp_mode_two_lanes && !tphy->dp_two_lanes) + t6040_dp_xbar_config(tphy, tphy->dp_flip); + t6040_usb3_reclaim_locked(tphy); + return 0; +} + +static void t6040_dp_mux_unregister(void *mux) +{ + typec_mux_unregister(mux); +} + +/* + * Everything the DP route needs from the device tree; a node without it gets + * no DP phy. A USB-C PHY offers DP through a Type-C mode switch; a fixed DP + * route (an HDMI converter's PHY) is driven straight by the DCP with no mux + * and uses the normal lane map. + */ +static int t6040_dp_probe(struct t6040_usb2_phy *tphy) +{ + struct device *dev = tphy->dev; + unsigned int i; + + tphy->dp_rate = -1; + + /* + * DP mode needs the "core" and "cfg" windows. A USB-C PHY has them + * alongside its USB regions; a fixed DP PHY has them without any + * pipehandler or USB3 coprocessor. A node with neither is USB2 only. + */ + if (!tphy->regs[T6040_USB3_CORE] || !tphy->regs[T6040_USB3_CFG]) + return 0; + + tphy->dp = devm_phy_create(dev, NULL, &t6040_dp_phy_ops); + if (IS_ERR(tphy->dp)) + return PTR_ERR(tphy->dp); + phy_set_drvdata(tphy->dp, tphy); + + /* DP-only node: the generic apply_tunables tables (absent = not applied). */ + if (t6040_dp_only(tphy)) { + for (i = 0; i < T6040_DP_NR_GENERIC_TUNABLES; i++) { + const char *prop = t6040_usb3_tunable_list[i].prop; + + if (!of_property_present(dev->of_node, prop) || + !tphy->res[t6040_usb3_tunable_list[i].region]) { + dev_warn(dev, "DP: no %s\n", prop); + tphy->tunables[i] = NULL; + continue; + } + struct resource *res = tphy->res[t6040_usb3_tunable_list[i].region]; + + tphy->tunables[i] = devm_apple_tunable_parse(dev, dev->of_node, prop, res); + if (IS_ERR(tphy->tunables[i])) { + dev_warn(dev, "DP: %s: %pe\n", prop, tphy->tunables[i]); + tphy->tunables[i] = NULL; + } + } + } + + if (of_property_present(dev->of_node, "mode-switch")) { + struct typec_mux_desc mux_desc = { + .drvdata = tphy, + .fwnode = dev_fwnode(dev), + .set = t6040_dp_mux_set, + }; + + tphy->mux = typec_mux_register(dev, &mux_desc); + if (IS_ERR(tphy->mux)) + return PTR_ERR(tphy->mux); + if (devm_add_action_or_reset(dev, t6040_dp_mux_unregister, tphy->mux)) + return -ENOMEM; + } + + tphy->dp_ok = true; + return 0; +} + +/* ---- the coprocessor's RTKit ---- */ + +/* The coprocessor places its buffers in its own __DATA SRAM. */ +static int t6040_cpu_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct t6040_usb2_phy *tphy = cookie; + resource_size_t size = resource_size(tphy->cpu_data); + + if (!bfr->iova || bfr->iova < T6040_CPU_DATA_IOVA || bfr->size > size || + bfr->iova - T6040_CPU_DATA_IOVA > size - bfr->size) { + dev_err(tphy->dev, "USB3: coprocessor buffer %pad+%#zx outside its SRAM\n", + &bfr->iova, bfr->size); + return -EINVAL; + } + bfr->iomem = ioremap_np(tphy->cpu_data->start + (bfr->iova - T6040_CPU_DATA_IOVA), + bfr->size); + if (!bfr->iomem) + return -ENOMEM; + bfr->is_mapped = true; + return 0; +} + +static void t6040_cpu_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + if (bfr->iomem) + iounmap(bfr->iomem); +} + +static void t6040_cpu_crashed(void *cookie, const void *crashlog, size_t size) +{ + struct t6040_usb2_phy *tphy = cookie; + + dev_err(tphy->dev, "USB3: the PHY coprocessor crashed\n"); +} + +static void t6040_cpu_recv_message(void *cookie, u8 ep, u64 msg) +{ + struct t6040_usb2_phy *tphy = cookie; + + if (ep == T6040_CPU_EP_APP) { + if (atomic_xchg(&tphy->drop_next_power_down_ack, 0)) { + dev_warn(tphy->dev, "USB3: dropped Power Down acknowledge (fault injection)\n"); + return; + } + dev_dbg(tphy->dev, "USB3: Power Down acknowledge %#llx\n", msg); + complete(&tphy->app_reply); + } else { + dev_dbg(tphy->dev, "USB3: coprocessor message on endpoint %u: %#llx\n", ep, msg); + } +} + +static const struct apple_rtkit_ops t6040_cpu_rtkit_ops = { + .flags = APPLE_RTKIT_COPROC_PLACES_BUFFERS, + .crashed = t6040_cpu_crashed, + .recv_message = t6040_cpu_recv_message, + .shmem_setup = t6040_cpu_shmem_setup, + .shmem_destroy = t6040_cpu_shmem_destroy, +}; + +static void t6040_rtkit_cleanup(void *data) +{ + t6040_rtkit_stop_transport(data); +} + +/* + * Whether the PHY's power domain has to stay on at a power-off that is about + * to happen: while the coprocessor's firmware runs. + * + * The PHY's domain, and the mailbox's above it, power the coprocessor. A + * system sleep takes the domain of every device that has suspended off, a + * suspend-to-idle included, which leaves a host running: its controller + * keeps its own domain as a syscore device, and nothing kept this one. The + * coprocessor then lost its power under a firmware the driver took for + * running, and did not acknowledge the idle request of its next stop. + */ +static bool t6040_pd_stays_on(unsigned long action, bool fw_live) +{ + return action == GENPD_NOTIFY_PRE_OFF && fw_live; +} + +/* Called by the PM domain with its lock held, which is a spinlock for these domains. */ +static int t6040_pd_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct t6040_usb2_phy *tphy = container_of(nb, struct t6040_usb2_phy, pd_nb); + + if (t6040_pd_stays_on(action, READ_ONCE(tphy->fw_live))) + return notifier_from_errno(-EBUSY); + return NOTIFY_OK; +} + +static void t6040_pd_notifier_remove(void *data) +{ + dev_pm_genpd_remove_notifier(data); +} + +/* Without a single PM domain nothing takes the coprocessor's power away under it. */ +static int t6040_pd_notifier_init(struct t6040_usb2_phy *tphy) +{ + int ret; + + tphy->pd_nb.notifier_call = t6040_pd_notify; + ret = dev_pm_genpd_add_notifier(tphy->dev, &tphy->pd_nb); + if (ret == -ENODEV || ret == -EOPNOTSUPP) + return 0; + if (ret) + return ret; + return devm_add_action_or_reset(tphy->dev, t6040_pd_notifier_remove, tphy->dev); +} + +static void t6040_mbox_dev_put(void *data) +{ + put_device(data); +} + +static int t6040_get_mbox_dev(struct t6040_usb2_phy *tphy) +{ + struct of_phandle_args args; + struct platform_device *pdev; + int ret; + + ret = of_parse_phandle_with_args(tphy->dev->of_node, "mboxes", "#mbox-cells", 0, + &args); + if (ret) + return ret; + pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + if (!pdev) + return -EPROBE_DEFER; + + tphy->mbox_dev = &pdev->dev; + return devm_add_action_or_reset(tphy->dev, t6040_mbox_dev_put, tphy->mbox_dev); +} + +static void t6040_debugfs_remove(void *data) +{ + debugfs_remove_recursive(data); +} + +static int t6040_debugfs_init(struct t6040_usb2_phy *tphy) +{ + const char *name; + int ret; + + name = devm_kasprintf(tphy->dev, GFP_KERNEL, "t6040-atcphy-%s", + dev_name(tphy->dev)); + if (!name) + return -ENOMEM; + tphy->debugfs = debugfs_create_dir(name, NULL); + if (IS_ERR(tphy->debugfs)) { + if (PTR_ERR(tphy->debugfs) == -ENODEV) { + tphy->debugfs = NULL; + return 0; + } + return PTR_ERR(tphy->debugfs); + } + debugfs_create_atomic_t("drop_next_power_down_ack", 0600, tphy->debugfs, + &tphy->drop_next_power_down_ack); + ret = devm_add_action_or_reset(tphy->dev, t6040_debugfs_remove, tphy->debugfs); + if (ret) + tphy->debugfs = NULL; + return ret; +} + +/* + * Map the PHY register windows shared by the USB3 and DisplayPort paths. Each + * is optional: a USB-C PHY has the full set, the fixed HDMI DP PHY (atcphy3) + * lacks none of core/cfg but has no pipehandler. Returns true once at least + * "core" and "cfg" are mapped, which is all DP mode needs. + */ +static bool t6040_map_regions(struct t6040_usb2_phy *tphy, struct platform_device *pdev) +{ + struct device *dev = tphy->dev; + size_t i; + + for (i = 0; i < T6040_USB3_NR_REGIONS; i++) { + tphy->res[i] = platform_get_resource_byname(pdev, IORESOURCE_MEM, + t6040_usb3_region_names[i]); + if (!tphy->res[i]) + continue; + tphy->regs[i] = devm_ioremap_resource(dev, tphy->res[i]); + if (IS_ERR(tphy->regs[i])) { + dev_warn(dev, "\"%s\" region: %pe\n", + t6040_usb3_region_names[i], tphy->regs[i]); + tphy->regs[i] = NULL; + tphy->res[i] = NULL; + } + } + return tphy->regs[T6040_USB3_CORE] && tphy->regs[T6040_USB3_CFG]; +} + +/* + * Everything USB3 needs from the device tree. A missing piece leaves the + * port at USB2; only a mailbox that has not probed yet defers. + */ +static int t6040_usb3_probe(struct t6040_usb2_phy *tphy, struct platform_device *pdev) +{ + struct device *dev = tphy->dev; + struct typec_switch_desc sw_desc = { + .drvdata = tphy, + .fwnode = dev_fwnode(dev), + .set = t6040_usb3_sw_set, + }; + struct typec_switch_dev *sw; + struct resource *ph; + size_t i; + int ret; + + tphy->usb3 = devm_phy_create(dev, NULL, &t6040_usb3_phy_ops); + if (IS_ERR(tphy->usb3)) + return PTR_ERR(tphy->usb3); + phy_set_drvdata(tphy->usb3, tphy); + init_completion(&tphy->app_reply); + + if (!of_property_present(dev->of_node, "orientation-switch")) + return 0; + + /* + * Registered first: tipd waits for the switch the DT names, so the port + * keeps USB2, roles and PD even if a USB3 piece below is missing. + */ + sw = typec_switch_register(dev, &sw_desc); + if (IS_ERR(sw)) + return PTR_ERR(sw); + ret = devm_add_action_or_reset(dev, t6040_usb3_switch_unregister, sw); + if (ret) + return ret; + + for (i = 0; i < T6040_USB3_NR_REGIONS; i++) { + if (!tphy->regs[i]) { + dev_warn(dev, "USB3: no \"%s\" region, USB2 only\n", + t6040_usb3_region_names[i]); + return 0; + } + } + tphy->cpu_data = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpu-data"); + ph = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pipehandler"); + if (!tphy->cpu_data || !ph) { + dev_warn(dev, "USB3: no coprocessor SRAM region, USB2 only\n"); + return 0; + } + + for (i = 0; i < T6040_USB3_NR_TUNABLES; i++) { + tphy->tunables[i] = + devm_apple_tunable_parse(dev, dev->of_node, t6040_usb3_tunable_list[i].prop, + tphy->res[t6040_usb3_tunable_list[i].region]); + if (IS_ERR(tphy->tunables[i])) { + dev_warn(dev, "USB3: %s: %pe, USB2 only\n", + t6040_usb3_tunable_list[i].prop, tphy->tunables[i]); + tphy->tunables[i] = NULL; + return 0; + } + } + tphy->tunable_pipehandler = devm_apple_tunable_parse(dev, dev->of_node, + "apple,tunable-pipehandler", ph); + if (IS_ERR(tphy->tunable_pipehandler)) { + dev_warn(dev, "USB3: apple,tunable-pipehandler: %pe, USB2 only\n", + tphy->tunable_pipehandler); + return 0; + } + + ret = t6040_get_mbox_dev(tphy); + if (ret) + return dev_err_probe(dev, ret, "failed to find PHY coprocessor mailbox\n"); tphy->rtk = apple_rtkit_init(dev, tphy, "phy-cpu", 0, &t6040_cpu_rtkit_ops); if (IS_ERR(tphy->rtk)) { @@ -2296,11 +3160,45 @@ static int t6040_usb3_probe(struct t6040_usb2_phy *tphy, struct platform_device if (ret) return ret; - ret = t6040_debugfs_init(tphy); - if (ret) - return ret; + ret = t6040_debugfs_init(tphy); + if (ret) + return ret; + + ret = t6040_pd_notifier_init(tphy); + if (ret) + return dev_err_probe(dev, ret, "failed to follow the PHY's power domain\n"); + + tphy->usb3_ok = true; + return 0; +} + +/* The controller's reset, see t6040_usb2_reset_assert(). */ +static void t6040_dwc3_reset_hold_locked(struct t6040_usb2_phy *tphy) +{ + lockdep_assert_held(&tphy->lock); + t6040_usb2_clear32(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN, + T6040_PIPEHANDLER_AON_GEN_DWC3_RESET_N); + t6040_usb2_set32(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN, + T6040_PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); +} + +/* -EIO if the register does not read back released: the controller may not be there. */ +static int t6040_dwc3_reset_release_locked(struct t6040_usb2_phy *tphy) +{ + u32 value; - tphy->usb3_ok = true; + lockdep_assert_held(&tphy->lock); + t6040_usb2_clear32(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN, + T6040_PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + t6040_usb2_set32(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN, + T6040_PIPEHANDLER_AON_GEN_DWC3_RESET_N); + + value = readl(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN); + if (value == U32_MAX || + (value & (T6040_PIPEHANDLER_AON_GEN_DWC3_RESET_N | + T6040_PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN)) != + T6040_PIPEHANDLER_AON_GEN_DWC3_RESET_N) + return -EIO; return 0; } @@ -2323,10 +3221,9 @@ static int t6040_usb2_reset_assert(struct reset_controller_dev *rcdev, return ret; guard(mutex)(&tphy->lock); - t6040_usb2_clear32(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN, - T6040_PIPEHANDLER_AON_GEN_DWC3_RESET_N); - t6040_usb2_set32(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN, - T6040_PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + t6040_dwc3_reset_hold_locked(tphy); + /* From here the reset is its consumer's again, whatever a sleep did with it. */ + tphy->sleep_reset = false; if (tphy->dwc3_running) { tphy->dwc3_running = false; pm_runtime_put(tphy->dev); @@ -2334,166 +3231,699 @@ static int t6040_usb2_reset_assert(struct reset_controller_dev *rcdev, return 0; } -static int t6040_usb2_reset_deassert(struct reset_controller_dev *rcdev, - unsigned long id) +static int t6040_usb2_reset_deassert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct t6040_usb2_phy *tphy = + container_of(rcdev, struct t6040_usb2_phy, rcdev); + int ret; + + PM_RUNTIME_ACQUIRE(tphy->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + guard(mutex)(&tphy->lock); + if (tphy->shutdown_failed) + return -EIO; + ret = t6040_dwc3_reset_release_locked(tphy); + if (ret) + return ret; + tphy->sleep_reset = false; + if (!tphy->dwc3_running) { + pm_runtime_get_noresume(tphy->dev); + tphy->dwc3_running = true; + } + return 0; +} + +static const struct reset_control_ops t6040_usb2_reset_ops = { + .assert = t6040_usb2_reset_assert, + .deassert = t6040_usb2_reset_deassert, +}; + +static int t6040_usb2_reset_xlate(struct reset_controller_dev *rcdev, + const struct of_phandle_args *reset_spec) +{ + return 0; +} + +static struct phy *t6040_usb2_xlate(struct device *dev, + const struct of_phandle_args *args) +{ + struct t6040_usb2_phy *tphy = dev_get_drvdata(dev); + + if (args->args_count != 1) + return ERR_PTR(-ENODEV); + + switch (args->args[0]) { + case PHY_TYPE_USB2: + return tphy->phy; + case PHY_TYPE_USB3: + return tphy->usb3; + case PHY_TYPE_DP: + return tphy->dp; + default: + return ERR_PTR(-ENODEV); + } +} + +static int t6040_usb2_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct t6040_usb2_phy *tphy; + struct phy_provider *provider; + int ret; + + tphy = devm_kzalloc(dev, sizeof(*tphy), GFP_KERNEL); + if (!tphy) + return -ENOMEM; + + tphy->dev = dev; + /* + * dwc3 powers on its phys from dwc3_core_init(), which runs inside + * dwc3_core_probe() -- before any consumer can deliver phy_set_mode() + * to this phy. dwc3-apple's early phy_set_mode() call happens while + * dwc.usb2_generic_phy[] is still unpopulated, so it is a silent no-op + * on a NULL phy, and a PHY_MODE_INVALID default then fails power_on + * with -EINVAL ("failed to initialize core", ticket 108). This + * provider implements exactly one mode, so start in it; set_mode + * still rejects every request that is not PHY_MODE_USB_HOST. + */ + tphy->mode = PHY_MODE_USB_HOST; + mutex_init(&tphy->lock); + init_waitqueue_head(&tphy->sleep_wq); + atomic_set(&tphy->drop_next_power_down_ack, 0); + platform_set_drvdata(pdev, tphy); + + tphy->usb2 = devm_platform_ioremap_resource_byname(pdev, "bank0"); + if (IS_ERR(tphy->usb2)) + return dev_err_probe(dev, PTR_ERR(tphy->usb2), + "failed to map USB2 bank 0\n"); + + if (of_property_present(dev->of_node, "apple,tunable-usb2-host")) { + tphy->tunable_usb2_host = + devm_apple_tunable_parse(dev, dev->of_node, "apple,tunable-usb2-host", + platform_get_resource_byname(pdev, IORESOURCE_MEM, + "bank0")); + if (IS_ERR(tphy->tunable_usb2_host)) + return dev_err_probe(dev, PTR_ERR(tphy->tunable_usb2_host), + "invalid apple,tunable-usb2-host\n"); + } + + tphy->evt = devm_platform_ioremap_resource_byname(pdev, "bank1"); + if (IS_ERR(tphy->evt)) + return dev_err_probe(dev, PTR_ERR(tphy->evt), + "failed to map USB2 event bank 1\n"); + + /* Before devm_phy_create(): the PHY core then manages the references. */ + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + + tphy->phy = devm_phy_create(dev, NULL, &apple_t6040_usb2_phy_ops); + if (IS_ERR(tphy->phy)) + return PTR_ERR(tphy->phy); + phy_set_drvdata(tphy->phy, tphy); + + /* + * The USB-C PHYs have a pipehandler and drive a dwc3 (reset provider); + * the fixed HDMI DP PHY (atcphy3) has neither. Map it and register the + * reset controller only when present, so a DP-only node still probes. + */ + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "pipehandler")) { + tphy->pipehandler = devm_platform_ioremap_resource_byname(pdev, + "pipehandler"); + if (IS_ERR(tphy->pipehandler)) + return dev_err_probe(dev, PTR_ERR(tphy->pipehandler), + "failed to map pipehandler bank\n"); + + tphy->rcdev.owner = THIS_MODULE; + tphy->rcdev.nr_resets = 1; + tphy->rcdev.ops = &t6040_usb2_reset_ops; + tphy->rcdev.of_node = dev->of_node; + tphy->rcdev.of_reset_n_cells = 0; + tphy->rcdev.of_xlate = t6040_usb2_reset_xlate; + ret = devm_reset_controller_register(dev, &tphy->rcdev); + if (ret) + return ret; + } + + /* Map the core/cfg (and USB3) windows once; both sub-probes share them. */ + t6040_map_regions(tphy, pdev); + + ret = t6040_usb3_probe(tphy, pdev); + if (ret) + return ret; + + ret = t6040_cio_probe(tphy); + if (ret) + return ret; + + ret = t6040_dp_probe(tphy); + if (ret) + return ret; + + provider = devm_of_phy_provider_register(dev, t6040_usb2_xlate); + return PTR_ERR_OR_ZERO(provider); +} + +/* ---- a sleep with the host kept (linux/phy/apple-atc.h) ---- + * + * A deep sleep powers the SoC down, and with it this PHY, the controller and + * its xHC. dwc3-apple used to stop the host for it, which removes every USB + * device, and start it again afterwards. For a port that only does USB it + * keeps the xHCI device and its USB devices instead, and the PHY goes through + * the same stop and start below them: down once xHCI has suspended + * (apple_t6040_atcphy_sleep_enter()), the controller out of reset before its + * core is touched again (apple_t6040_atcphy_sleep_result()), USB2 and USB3 back + * once the core is set up (apple_t6040_atcphy_sleep_power_up()). xHCI then + * starts over and the USB core resets the devices it finds again. + * + * The register sequences are those of power_off, power_on and the reset, in + * the order dwc3-apple's stop and start run them, so the CFG block is parked + * at the sleep's commit as after any detach. The PHY's firmware can save its + * state for a sleep and restore it at the wake instead; the devices stay + * without that, and it isn't done here. The steps are the same whether or + * not the SoC then loses power (a refused sleep, a dry run), and leave + * nothing the wake doesn't set up again. + * + * The phys' power counts and the reset's runtime PM reference are their + * consumers' and stay as they are; sleep_down and sleep_reset say what the + * hardware is behind them. A host that stops after all goes through the phys' + * and the reset's own operations, which end that. + */ + +static struct t6040_usb2_phy *t6040_from_usb2_phy(struct phy *phy) +{ + if (!phy || phy->ops != &apple_t6040_usb2_phy_ops) + return NULL; + return phy_get_drvdata(phy); +} + +/* + * A USB2 host that runs, alone or with USB3 on this PHY and the PIPE on it. + * DisplayPort on the lanes and USB4/Thunderbolt have clients of their own, + * which come back in their own order: such a port's host stops for the + * sleep. So does one whose four lanes are DisplayPort's by its pin assignment + * (C, E), link or not. + * + * Pin assignment D without a DisplayPort link is a USB port to the PHY: a + * dock with no display on it. USB3 runs on its pair as on any other port, + * nothing is programmed for the other pair before the DCP powers the DP phy + * on, and the mode is only latched here for that. With a link up the port + * stops its host like the others. + */ +static bool t6040_can_sleep_in_place(const struct t6040_usb2_phy *tphy) +{ + return tphy->pipehandler && tphy->powered && tphy->dwc3_running && + tphy->mode == PHY_MODE_USB_HOST && !tphy->shutdown_failed && + !tphy->sleep_no_keep && !tphy->sleep_down && !tphy->sleep_reset && + !tphy->cio_up && !tphy->pipe_tunneled && !tphy->router_running && + !tphy->dp_up && (!tphy->dp_alt || tphy->dp_two_lanes) && + !tphy->usb3_lent && tphy->usb3_up == tphy->pipe_switched; +} + +/* Room for every name of t6040_sleep_in_place_misses() at once. */ +#define T6040_SLEEP_MISSES_LEN 192 + +/* For the log: the conditions of t6040_can_sleep_in_place() that fail, by name. */ +static void t6040_sleep_in_place_misses(const struct t6040_usb2_phy *tphy, char *buf, size_t size) +{ + int len = 0; + +#define T6040_MISS(cond, name) \ + do { if (cond) len += scnprintf(buf + len, size - len, " %s", name); } while (0) + buf[0] = 0; + T6040_MISS(!tphy->pipehandler, "no_pipehandler"); + T6040_MISS(!tphy->powered, "usb2_off"); + T6040_MISS(!tphy->dwc3_running, "dwc3_in_reset"); + T6040_MISS(tphy->mode != PHY_MODE_USB_HOST, "mode"); + T6040_MISS(tphy->shutdown_failed, "shutdown_failed"); + T6040_MISS(tphy->sleep_no_keep, "sleep_no_keep"); + T6040_MISS(tphy->sleep_down, "sleep_down"); + T6040_MISS(tphy->sleep_reset, "sleep_reset"); + T6040_MISS(tphy->cio_up, "cio_up"); + T6040_MISS(tphy->pipe_tunneled, "pipe_tunneled"); + T6040_MISS(tphy->router_running, "router_running"); + T6040_MISS(tphy->dp_up, "dp_up"); + T6040_MISS(tphy->dp_alt && !tphy->dp_two_lanes, "dp_four_lanes"); + T6040_MISS(tphy->usb3_lent, "usb3_lent"); + T6040_MISS(tphy->usb3_up != tphy->pipe_switched, "pipe_state"); +#undef T6040_MISS +} + +int apple_t6040_atcphy_sleep_prepare(struct phy *usb2_phy) +{ + struct t6040_usb2_phy *tphy = t6040_from_usb2_phy(usb2_phy); + + if (!tphy) + return -EOPNOTSUPP; + + guard(mutex)(&tphy->lock); + if (!t6040_can_sleep_in_place(tphy)) { + char why[T6040_SLEEP_MISSES_LEN]; + + t6040_sleep_in_place_misses(tphy, why, sizeof(why)); + dev_info(tphy->dev, + "the host stops for the sleep: USB2 %d, USB3 %d, PIPE %d, DP %d/%d, CIO %d, lent %d, failed %d/%d, because:%s\n", + tphy->powered, tphy->usb3_up, tphy->pipe_switched, tphy->dp_alt, + tphy->dp_up, tphy->cio_up, tphy->usb3_lent, tphy->shutdown_failed, + tphy->sleep_no_keep, why); + return -EBUSY; + } + tphy->sleep_armed = true; + tphy->sleep_ret = 0; + return 0; +} +EXPORT_SYMBOL_GPL(apple_t6040_atcphy_sleep_prepare); + +int apple_t6040_atcphy_sleep_enter(struct phy *usb2_phy) +{ + struct t6040_usb2_phy *tphy = t6040_from_usb2_phy(usb2_phy); + bool usb3; + int ret; + + if (!tphy) + return -EOPNOTSUPP; + + PM_RUNTIME_ACQUIRE(tphy->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + guard(mutex)(&tphy->lock); + if (!tphy->sleep_armed) + return -EINVAL; + if (tphy->sleep_down) + return 0; + /* The mux moved since the prepare: that mode's clients don't come back this way. */ + if (!t6040_can_sleep_in_place(tphy)) { + char why[T6040_SLEEP_MISSES_LEN]; + + t6040_sleep_in_place_misses(tphy, why, sizeof(why)); + dev_err(tphy->dev, + "USB changed for the sleep (USB3 %d, PIPE %d, DP %d/%d, CIO %d, because:%s): refusing it\n", + tphy->usb3_up, tphy->pipe_switched, tphy->dp_alt, tphy->dp_up, + tphy->cio_up, why); + tphy->sleep_ret = -EBUSY; + return -EBUSY; + } + + /* + * USB3 that an earlier wake of this host left down counts as well: this + * sleep's wake tries it again, with the SoC's power cycle behind it if + * the sleep is a real one. + */ + usb3 = tphy->usb3_up; + tphy->sleep_usb3 |= usb3; + /* As dwc3-apple's stop: USB3 with its PIPE, USB2, then the controller's reset. */ + t6040_usb3_down_locked(tphy); + ret = t6040_usb2_off_locked(tphy); + /* + * No reason to refuse the sleep: one that powers the SoC down resets the + * USB2 PHY whatever a write that didn't land has left of it, and the wake + * starts from there (t6040_power_lost()). After any other sleep the PHY + * stays failed, and the wake says so. + */ + if (ret) + dev_err(tphy->dev, "USB2 did not power down for the sleep: %d\n", ret); + t6040_dwc3_reset_hold_locked(tphy); + tphy->sleep_down = true; + tphy->sleep_reset = true; + WRITE_ONCE(tphy->sleep_back, false); + dev_info(tphy->dev, "PHY down for the sleep with the host kept, USB3 %s\n", + usb3 ? "was up" : "was off"); + return 0; +} +EXPORT_SYMBOL_GPL(apple_t6040_atcphy_sleep_enter); + +/* -ENODEV while the reset does not read back released: the core must not be touched. */ +static int t6040_sleep_release_locked(struct t6040_usb2_phy *tphy) +{ + lockdep_assert_held(&tphy->lock); + if (t6040_dwc3_reset_release_locked(tphy)) { + dev_err(tphy->dev, "the controller's reset did not release after the sleep: %#x\n", + readl(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN)); + tphy->sleep_ret = -ENODEV; + return -ENODEV; + } + tphy->sleep_reset = false; + tphy->sleep_ret = APPLE_ATCPHY_SLEEP_RESET; + return APPLE_ATCPHY_SLEEP_RESET; +} + +int apple_t6040_atcphy_sleep_result(struct phy *usb2_phy) +{ + struct t6040_usb2_phy *tphy = t6040_from_usb2_phy(usb2_phy); + int ret; + + if (!tphy) + return -EOPNOTSUPP; + + PM_RUNTIME_ACQUIRE(tphy->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + + guard(mutex)(&tphy->lock); + if (!tphy->sleep_armed) + return -ENODATA; + /* The first call after the sleep: without the PHY's power the reset can't release. */ + if (tphy->sleep_reset && ret) + tphy->sleep_ret = -ENODEV; + else if (tphy->sleep_reset) + t6040_sleep_release_locked(tphy); + return tphy->sleep_ret; +} +EXPORT_SYMBOL_GPL(apple_t6040_atcphy_sleep_result); + +/* + * Whether the wake of a sleep with the host kept starts USB3: it ran when the + * port went down, and the coprocessor doesn't wait for a power cycle of its + * domains, which a system resume can't give it. A sleep that powered the SoC + * down was that power cycle, see t6040_usb2_resume_early(). + */ +static bool t6040_sleep_wake_usb3(const struct t6040_usb2_phy *tphy) +{ + return tphy->sleep_usb3 && !tphy->recovery.cycle_pending; +} + +int apple_t6040_atcphy_sleep_power_up(struct phy *usb2_phy) { - struct t6040_usb2_phy *tphy = - container_of(rcdev, struct t6040_usb2_phy, rcdev); - u32 value; + struct t6040_usb2_phy *tphy = t6040_from_usb2_phy(usb2_phy); + bool tried; int ret; + if (!tphy) + return -EOPNOTSUPP; + PM_RUNTIME_ACQUIRE(tphy->dev, pm); ret = PM_RUNTIME_ACQUIRE_ERR(&pm); if (ret) return ret; guard(mutex)(&tphy->lock); - if (tphy->shutdown_failed) - return -EIO; - t6040_usb2_clear32(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN, - T6040_PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); - t6040_usb2_set32(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN, - T6040_PIPEHANDLER_AON_GEN_DWC3_RESET_N); + /* Only for a controller this sleep's result has taken out of reset. */ + if (!tphy->sleep_armed || tphy->sleep_reset) + return -EINVAL; + /* Whatever this comes to, DP has nothing more to wait for. */ + t6040_sleep_back_done(tphy); + if (!tphy->sleep_down) + return tphy->sleep_ret < 0 ? tphy->sleep_ret : 0; - value = readl(tphy->pipehandler + T6040_PIPEHANDLER_AON_GEN); - if (value == U32_MAX || - (value & (T6040_PIPEHANDLER_AON_GEN_DWC3_RESET_N | - T6040_PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN)) != - T6040_PIPEHANDLER_AON_GEN_DWC3_RESET_N) - return -EIO; - if (!tphy->dwc3_running) { - pm_runtime_get_noresume(tphy->dev); - tphy->dwc3_running = true; + /* As dwc3-apple's start, in the core's init: USB2, then USB3 if it ran. */ + ret = tphy->shutdown_failed ? -EIO : t6040_usb2_on_locked(tphy); + if (ret) { + dev_err(tphy->dev, "USB2 not back after the sleep (%d): the host has to restart\n", + ret); + tphy->sleep_ret = ret; + return ret; + } + tphy->sleep_down = false; + tried = t6040_sleep_wake_usb3(tphy); + if (tried) { + tphy->sleep_waking = true; + t6040_usb3_up_locked(tphy); + tphy->sleep_waking = false; } + /* + * As at a start, a port whose USB3 doesn't come up stays at USB2: the host + * is kept with what runs at USB2 on it, and only the SuperSpeed devices + * leave the bus. + */ + if (tphy->sleep_usb3 && !tphy->usb3_up) + dev_warn(tphy->dev, "USB3 not back after the sleep (%s): the host stays, at USB2\n", + tried ? "its bring-up failed" : "its coprocessor waits for a power cycle"); + else + dev_info(tphy->dev, "PHY back after the sleep with the host kept, USB3 %s\n", + tphy->usb3_up ? "up" : "off"); return 0; } +EXPORT_SYMBOL_GPL(apple_t6040_atcphy_sleep_power_up); -static const struct reset_control_ops t6040_usb2_reset_ops = { - .assert = t6040_usb2_reset_assert, - .deassert = t6040_usb2_reset_deassert, -}; - -static int t6040_usb2_reset_xlate(struct reset_controller_dev *rcdev, - const struct of_phandle_args *reset_spec) +int apple_t6040_atcphy_sleep_recover(struct phy *usb2_phy) { - return 0; -} + struct t6040_usb2_phy *tphy = t6040_from_usb2_phy(usb2_phy); -static struct phy *t6040_usb2_xlate(struct device *dev, - const struct of_phandle_args *args) -{ - struct t6040_usb2_phy *tphy = dev_get_drvdata(dev); + if (!tphy) + return -EOPNOTSUPP; - if (args->args_count != 1) - return ERR_PTR(-ENODEV); + PM_RUNTIME_ACQUIRE(tphy->dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) + return -ENODEV; - switch (args->args[0]) { - case PHY_TYPE_USB2: - return tphy->phy; - case PHY_TYPE_USB3: - return tphy->usb3; - case PHY_TYPE_DP: - return tphy->dp; - default: - return ERR_PTR(-ENODEV); - } + guard(mutex)(&tphy->lock); + /* Not held in reset by a sleep: there is nothing of this PHY's to put right. */ + if (!tphy->sleep_reset) + return -ENODEV; + return t6040_sleep_release_locked(tphy); } +EXPORT_SYMBOL_GPL(apple_t6040_atcphy_sleep_recover); -static int t6040_usb2_probe(struct platform_device *pdev) +/* + * Whether a sleep ended with keeping the host having failed: the host was + * lost, and what the wake came to is an error of the PHY's own, which is the + * controller's reset that didn't release or USB2 that didn't come back. USB3 + * never costs the host. A sleep refused for a USB change doesn't count, nor + * a host that was restarted for another reason. + */ +static bool t6040_sleep_keep_failed(bool kept, int sleep_ret) { - struct device *dev = &pdev->dev; - struct t6040_usb2_phy *tphy; - struct phy_provider *provider; - int ret; + return !kept && sleep_ret < 0 && sleep_ret != -EBUSY; +} + +void apple_t6040_atcphy_sleep_finish(struct phy *usb2_phy, bool kept) +{ + struct t6040_usb2_phy *tphy = t6040_from_usb2_phy(usb2_phy); - tphy = devm_kzalloc(dev, sizeof(*tphy), GFP_KERNEL); if (!tphy) - return -ENOMEM; + return; - tphy->dev = dev; + guard(mutex)(&tphy->lock); + t6040_sleep_back_done(tphy); + if (!tphy->sleep_armed) + return; /* - * dwc3 powers on its phys from dwc3_core_init(), which runs inside - * dwc3_core_probe() -- before any consumer can deliver phy_set_mode() - * to this phy. dwc3-apple's early phy_set_mode() call happens while - * dwc.usb2_generic_phy[] is still unpopulated, so it is a silent no-op - * on a NULL phy, and a PHY_MODE_INVALID default then fails power_on - * with -EINVAL ("failed to initialize core", ticket 108). This - * provider implements exactly one mode, so start in it; set_mode - * still rejects every request that is not PHY_MODE_USB_HOST. + * The host has been restarted by now. It stops for the next sleep, and + * the start after that one lets it be kept again. */ - tphy->mode = PHY_MODE_USB_HOST; - mutex_init(&tphy->lock); - atomic_set(&tphy->drop_next_power_down_ack, 0); - platform_set_drvdata(pdev, tphy); + if (t6040_sleep_keep_failed(kept, tphy->sleep_ret)) { + tphy->sleep_no_keep = true; + dev_warn(tphy->dev, + "PHY did not come back from a sleep with the host kept (%d): the host stops for the next sleep\n", + tphy->sleep_ret); + } + tphy->sleep_armed = false; + tphy->sleep_ret = 0; +} +EXPORT_SYMBOL_GPL(apple_t6040_atcphy_sleep_finish); - tphy->usb2 = devm_platform_ioremap_resource_byname(pdev, "bank0"); - if (IS_ERR(tphy->usb2)) - return dev_err_probe(dev, PTR_ERR(tphy->usb2), - "failed to map USB2 bank 0\n"); +/* + * The SoC lost power in a sleep or for a resume from disk, and the PHY with + * it. That was the power cycle its coprocessor may have been waiting for, and + * it put the USB2 PHY back into its reset state: a power-down or a start that + * didn't land before it says nothing about the PHY any more. + */ +static void __maybe_unused t6040_power_lost(struct t6040_usb2_phy *tphy) +{ + t6040_recovery_power_lost(&tphy->recovery); + tphy->shutdown_failed = false; + WRITE_ONCE(tphy->fw_live, false); +} - if (of_property_present(dev->of_node, "apple,tunable-usb2-host")) { - tphy->tunable_usb2_host = - devm_apple_tunable_parse(dev, dev->of_node, "apple,tunable-usb2-host", - platform_get_resource_byname(pdev, IORESOURCE_MEM, - "bank0")); - if (IS_ERR(tphy->tunable_usb2_host)) - return dev_err_probe(dev, PTR_ERR(tphy->tunable_usb2_host), - "invalid apple,tunable-usb2-host\n"); +#ifdef CONFIG_HIBERNATION +static int t6040_phy_count(struct phy *phy) +{ + return IS_ERR_OR_NULL(phy) ? 0 : phy->power_count; +} + +/* + * Resume from disk is a cold boot: the PHY, its coprocessor and the + * pipehandler come back reset, while this driver's state comes from the + * image. Before the image its consumers gave it back: the sleep hook + * unbinds dwc3 (USB2/USB3 phys powered off, controller reset asserted), + * the DCP powers the DP phy off at hibernation prepare. What is left set + * here describes hardware that no longer is: clear it to the state after + * probe, so that the next power-on programs the PHY afresh (with pin D, + * t6040_dp_power_on() otherwise takes the USB3 PHY for running). What + * consumers hold (powered, usb3_on, usb3_pipe, dwc3_running and the PHY + * core counts, with their runtime PM reference) is theirs to release and + * is kept, so that every count stays balanced; a phy still held is only + * reported. The port's Type-C state (orientation, DP alt mode, pin + * assignment) describes the partner, not the PHY, and stays for the next + * power-on. A power cycle that the coprocessor was waiting for when the + * image was made has happened by now. Early, before any .restore: the + * Type-C controller's may set the mux again. Nothing touches the hardware + * here. CIO is torn down before the image (.freeze), so the image never + * holds a CIO session. + */ +static int t6040_usb2_restore_early(struct device *dev) +{ + struct t6040_usb2_phy *tphy = dev_get_drvdata(dev); + + if (!tphy) + return 0; + guard(mutex)(&tphy->lock); + if (tphy->powered || tphy->usb3_on || tphy->dwc3_running || + t6040_phy_count(tphy->dp)) + dev_warn(dev, + "restore: still held across the image (usb2 %d, usb3 %d, dwc3 %d, dp %d/%d): its next power-on finds the PHY cold\n", + tphy->powered, tphy->usb3_on, tphy->dwc3_running, + t6040_phy_count(tphy->dp), tphy->dp_up); + /* .freeze refuses an image with CIO up; never trust one that has it. */ + if (tphy->cio_up) { + dev_warn(dev, "restore: CIO was up in the image, dropped\n"); + tphy->cio_up = false; + tphy->router_running = false; + tphy->usb3_tunnel_req = false; + tphy->pipe_tunneled = false; + tphy->tun_pll = false; + tphy->tun_rate[0] = 0; + tphy->tun_rate[1] = 0; + pm_runtime_put_noidle(dev); } + dev_info(dev, "restore: PHY state reset (usb3 up %d, pipe %d, lent %d, dp up %d, 2-lane %d)\n", + tphy->usb3_up, tphy->pipe_switched, tphy->usb3_lent, tphy->dp_up, + tphy->dp_mode_two_lanes); + tphy->usb3_up = false; + tphy->pipe_switched = false; + tphy->usb3_lent = false; + tphy->dp_up = false; + tphy->dp_mode_two_lanes = false; + tphy->dp_mode_flip = false; + tphy->dp_rate = -1; + t6040_power_lost(tphy); + return 0; +} - tphy->evt = devm_platform_ioremap_resource_byname(pdev, "bank1"); - if (IS_ERR(tphy->evt)) - return dev_err_probe(dev, PTR_ERR(tphy->evt), - "failed to map USB2 event bank 1\n"); +/* + * The coprocessor's RTKit session is reset with it. Its host side is + * reset at power-down already (t6040_usb3_firmware_power_down()); one + * still running here (USB3 held across the image) is reset now, in the + * normal phase, after the DARTs its buffers live behind are restored. + */ +static int t6040_usb2_restore(struct device *dev) +{ + struct t6040_usb2_phy *tphy = dev_get_drvdata(dev); + int ret; - /* Before devm_phy_create(): the PHY core then manages the references. */ - ret = devm_pm_runtime_enable(dev); - if (ret) - return ret; + if (!tphy || IS_ERR_OR_NULL(tphy->rtk)) + return 0; + guard(mutex)(&tphy->lock); + if (!apple_rtkit_is_running(tphy->rtk)) + return 0; + ret = apple_rtkit_reinit(tphy->rtk); + dev_info(dev, "restore: USB3 coprocessor RTKit reset: %d\n", ret); + return 0; +} - tphy->phy = devm_phy_create(dev, NULL, &t6040_usb2_phy_ops); - if (IS_ERR(tphy->phy)) - return PTR_ERR(tphy->phy); - phy_set_drvdata(tphy->phy, tphy); +#endif - /* - * The USB-C PHYs have a pipehandler and drive a dwc3 (reset provider); - * the fixed HDMI DP PHY (atcphy3) has neither. Map it and register the - * reset controller only when present, so a DP-only node still probes. - */ - if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "pipehandler")) { - tphy->pipehandler = devm_platform_ioremap_resource_byname(pdev, - "pipehandler"); - if (IS_ERR(tphy->pipehandler)) - return dev_err_probe(dev, PTR_ERR(tphy->pipehandler), - "failed to map pipehandler bank\n"); +#ifdef CONFIG_PM_SLEEP +/* + * System sleep with the port in USB4/Thunderbolt: s2idle, a firmware-driven + * suspend (pm_suspend_via_firmware(), which brings the PHY back like a cold + * start) and the three phases of hibernation that leave the running system + * (freeze before the image, poweroff after it). CIO is not kept across a + * sleep: the ACIO sleep is a full PHY shutdown, and + * the ACIO wake a full PHY init for mode 2; the firmware-assisted S2R state + * save is for the USB modes only. Here CIO is torn down the same way in each + * of them, after the router has stopped (apple_t6040_cio_router_stopped(), + * from its sleep notifier before any device callback); its DP tunnels and the + * tunnelled PIPE go with it. Nothing is kept for resume: the Type-C + * controller reads the port afresh after every sleep and sets the mux again, + * which brings CIO back before it starts the router. A router still marked + * running breaks that contract and fails the transition. Without a CIO + * session none of this touches the hardware. + */ +static int t6040_usb2_sleep(struct device *dev, const char *what) +{ + struct t6040_usb2_phy *tphy = dev_get_drvdata(dev); - tphy->rcdev.owner = THIS_MODULE; - tphy->rcdev.nr_resets = 1; - tphy->rcdev.ops = &t6040_usb2_reset_ops; - tphy->rcdev.of_node = dev->of_node; - tphy->rcdev.of_reset_n_cells = 0; - tphy->rcdev.of_xlate = t6040_usb2_reset_xlate; - ret = devm_reset_controller_register(dev, &tphy->rcdev); - if (ret) - return ret; + if (!tphy) + return 0; + guard(mutex)(&tphy->lock); + if (!tphy->cio_up) + return 0; + if (tphy->router_running) { + dev_err(dev, "%s: the USB4 router still runs on this port; it must stop before the PHY\n", + what); + return -EBUSY; } + t6040_cio_down_locked(tphy); + dev_info(dev, "%s: CIO down for system sleep; the Type-C controller brings it back\n", + what); + return 0; +} - /* Map the core/cfg (and USB3) windows once; both sub-probes share them. */ - t6040_map_regions(tphy, pdev); +static int t6040_usb2_suspend(struct device *dev) +{ + return t6040_usb2_sleep(dev, "suspend"); +} - ret = t6040_usb3_probe(tphy, pdev); - if (ret) - return ret; +static int t6040_usb2_freeze(struct device *dev) +{ + return t6040_usb2_sleep(dev, "freeze"); +} - ret = t6040_dp_probe(tphy); - if (ret) - return ret; +static int t6040_usb2_poweroff(struct device *dev) +{ + return t6040_usb2_sleep(dev, "poweroff"); +} - provider = devm_of_phy_provider_register(dev, t6040_usb2_xlate); - return PTR_ERR_OR_ZERO(provider); +/* + * A port that went down for the sleep with its host kept comes back in the + * host's resume. From here until then DP waits for it in its power-on, see + * t6040_dp_wait_host(). Early, before any .resume: the Type-C controller + * reports a display from its own. + * + * If the firmware woke the system, the SoC was powered down, and the PHY and + * its coprocessor with it: no start after this sleep has a power cycle to + * wait for or to run, and none finds the USB2 PHY failed by a write from + * before the sleep, the wake of a host kept through it included. + */ +static int t6040_usb2_resume_early(struct device *dev) +{ + struct t6040_usb2_phy *tphy = dev_get_drvdata(dev); + + if (!tphy) + return 0; + guard(mutex)(&tphy->lock); + if (pm_resume_via_firmware()) + t6040_power_lost(tphy); + WRITE_ONCE(tphy->sleep_back, tphy->sleep_down); + return 0; +} + +/* + * A sleep that is abandoned while devices still suspend has no early phase, + * and the host's resume brings a port that went down for it back all the + * same. This runs before that resume, the PHY being its supplier: DP waits + * from here. After any other sleep it finds what the early phase left. + */ +static int t6040_usb2_resume(struct device *dev) +{ + struct t6040_usb2_phy *tphy = dev_get_drvdata(dev); + + if (!tphy) + return 0; + guard(mutex)(&tphy->lock); + WRITE_ONCE(tphy->sleep_back, tphy->sleep_down); + return 0; } +static const struct dev_pm_ops t6040_usb2_pm_ops = { + .suspend = t6040_usb2_suspend, + .resume_early = t6040_usb2_resume_early, + .resume = t6040_usb2_resume, + .freeze = t6040_usb2_freeze, + .poweroff = t6040_usb2_poweroff, +#ifdef CONFIG_HIBERNATION + .restore_early = t6040_usb2_restore_early, + .restore = t6040_usb2_restore, +#endif +}; +#endif + static const struct of_device_id t6040_usb2_of_match[] = { { .compatible = "apple,t6040-atcphy" }, {} @@ -2505,8 +3935,85 @@ static struct platform_driver t6040_usb2_driver = { .driver = { .name = "phy-apple-t6040-usb2", .of_match_table = t6040_usb2_of_match, +#ifdef CONFIG_PM_SLEEP + .pm = &t6040_usb2_pm_ops, +#endif }, }; + +/* ---- the USB4 router's view of the PHY (linux/phy/apple-t6040-cio.h) ---- */ + +/* Run @fn on the PHY behind @phy with its lock held. */ +static int t6040_cio_call(struct phy *phy, + int (*fn)(struct t6040_usb2_phy *tphy, unsigned long arg), + unsigned long arg) +{ + struct t6040_usb2_phy *tphy = t6040_from_usb2_phy(phy); + + if (!tphy) + return -ENODEV; + + guard(mutex)(&tphy->lock); + return fn(tphy, arg); +} + +static int t6040_cio_state_fn(struct t6040_usb2_phy *tphy, unsigned long arg) +{ + return tphy->cio_up ? 1 : 0; +} + +static int t6040_cio_router_fn(struct t6040_usb2_phy *tphy, unsigned long running) +{ + if (running) { + if (!tphy->cio_up) + return -EAGAIN; + tphy->router_running = true; + dev_dbg(tphy->dev, "CIO: router started\n"); + t6040_cio_pipe_update_locked(tphy); + return 0; + } + if (!tphy->router_running) + return 0; + tphy->router_running = false; + tphy->usb3_tunnel_req = false; + t6040_cio_pipe_update_locked(tphy); + dev_dbg(tphy->dev, "CIO: router stopped\n"); + return 0; +} + +static int t6040_cio_usb3_fn(struct t6040_usb2_phy *tphy, unsigned long on) +{ + if (on && !tphy->router_running) + return -EAGAIN; + tphy->usb3_tunnel_req = on; + t6040_cio_pipe_update_locked(tphy); + return 0; +} + +int apple_t6040_cio_state(struct phy *phy) +{ + return t6040_cio_call(phy, t6040_cio_state_fn, 0); +} +EXPORT_SYMBOL_GPL(apple_t6040_cio_state); + +int apple_t6040_cio_router_started(struct phy *phy) +{ + return t6040_cio_call(phy, t6040_cio_router_fn, 1); +} +EXPORT_SYMBOL_GPL(apple_t6040_cio_router_started); + +void apple_t6040_cio_router_stopped(struct phy *phy) +{ + t6040_cio_call(phy, t6040_cio_router_fn, 0); +} +EXPORT_SYMBOL_GPL(apple_t6040_cio_router_stopped); + +int apple_t6040_cio_usb3_tunnel(struct phy *phy, bool on) +{ + return t6040_cio_call(phy, t6040_cio_usb3_fn, on); +} +EXPORT_SYMBOL_GPL(apple_t6040_cio_usb3_tunnel); + module_platform_driver(t6040_usb2_driver); MODULE_AUTHOR("The Asahi Linux Contributors"); diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index 3e8ab5a9b933b..3b686a0218940 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -54,8 +54,21 @@ struct apple_gpio_pinctrl { /* The pins' words across a firmware-driven suspend */ u32 *saved; unsigned long *saved_valid; + /* The regmap caches the block: it holds what this driver set, not the hardware. */ + bool cached; + /* A pin is written back in steps: see apple_gpio_safe_reconfigure(). */ + bool staged; /* Pins this driver has written. Others belong to someone else. */ unsigned long *touched; + /* + * A block whose every pin has to be put back after a sleep that lost its + * power (apple,resume-restore-all), but those in gpio-reserved-ranges. + */ + bool restore_all; + unsigned long *no_restore; + /* The block's word at REG_GLOBAL, kept across a sleep as well. */ + u32 saved_global; + bool saved_global_valid; struct apple_gpio_irq_group irqgrps[]; }; @@ -82,6 +95,12 @@ struct apple_gpio_pinctrl { #define REG_GPIOx_LOCK BIT(21) #define REG_GPIOx_DRIVE_STRENGTH1 GENMASK(23, 22) #define REG_IRQ(g, x) (0x800 + 0x40 * (g) + 4 * ((x) >> 5)) +/* + * One word past the pins and the interrupt groups. The block loses it with its + * power, and it has to be back before the pins are: with it lost, no pin reads + * its input any more. It is 1 on a running J616s. + */ +#define REG_GLOBAL 0xc48 static const struct regmap_config regmap_config = { .reg_bits = 32, @@ -403,6 +422,11 @@ struct apple_gpio_hw { * only within the valid GPIO registers minus gpio-reserved-ranges. */ bool no_regcache; + /* + * A pin written back after a sleep or a resume from disk takes its word in + * steps (apple_gpio_safe_reconfigure()). Elsewhere the word goes back whole. + */ + bool staged_restore; }; /* T8140 (A18 Pro) and T8132 (M4) both have faulting slots in this block. */ @@ -410,6 +434,11 @@ static const struct apple_gpio_hw apple_gpio_uncached_hw = { .no_regcache = true, }; +/* T6040 and T6041 (M4 Pro and Max) */ +static const struct apple_gpio_hw apple_gpio_t6041_hw = { + .staged_restore = true, +}; + static int apple_gpio_regmap_access(struct device *dev, unsigned int npins, struct regmap_config *config) { @@ -603,13 +632,36 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(pctl->map), "Failed to create regmap\n"); - /* Only with the regmap reaching the hardware: a cache would hide the loss. */ - if (hw && hw->no_regcache) { - pctl->saved = devm_kcalloc(&pdev->dev, npins, sizeof(*pctl->saved), GFP_KERNEL); - pctl->saved_valid = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); - pctl->touched = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); - if (!pctl->saved || !pctl->saved_valid || !pctl->touched) + /* + * A cache would hide the loss: on a cached block the resume compares and + * writes the hardware past it. + */ + pctl->cached = !(hw && hw->no_regcache); + pctl->staged = hw && hw->staged_restore; + pctl->saved = devm_kcalloc(&pdev->dev, npins, sizeof(*pctl->saved), GFP_KERNEL); + pctl->saved_valid = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + if (!pctl->saved || !pctl->saved_valid) + return -ENOMEM; + /* A resume from disk also puts pins back, from the cache where there is one. */ + pctl->touched = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + if (!pctl->touched) + return -ENOMEM; + pctl->restore_all = of_property_read_bool(pdev->dev.of_node, "apple,resume-restore-all"); + if (pctl->restore_all) { + int n = of_property_count_u32_elems(pdev->dev.of_node, "gpio-reserved-ranges"); + u32 start, count; + + pctl->no_restore = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + if (!pctl->no_restore) return -ENOMEM; + /* The pins the firmware lists as no-restore, as (first pin, count) pairs. */ + for (i = 0; n > 0 && i + 1 < (unsigned int)n; i += 2) + if (!of_property_read_u32_index(pdev->dev.of_node, "gpio-reserved-ranges", + i, &start) && + !of_property_read_u32_index(pdev->dev.of_node, "gpio-reserved-ranges", + i + 1, &count) && + start < npins && count <= npins - start) + bitmap_set(pctl->no_restore, start, count); } for (i = 0; i < npins; i++) { @@ -657,12 +709,44 @@ static const struct of_device_id apple_gpio_pinctrl_of_match[] = { { .compatible = "apple,t8140-pinctrl", .data = &apple_gpio_uncached_hw }, { .compatible = "apple,t8132-pinctrl", .data = &apple_gpio_uncached_hw }, { .compatible = "apple,t6050-pinctrl", .data = &apple_gpio_uncached_hw }, + { .compatible = "apple,t6041-pinctrl", .data = &apple_gpio_t6041_hw }, { .compatible = "apple,t8103-pinctrl", }, { .compatible = "apple,pinctrl", }, { } }; MODULE_DEVICE_TABLE(of, apple_gpio_pinctrl_of_match); +/* + * A pin's whole word, in the order the pin takes it after a power loss: the + * configuration first with the input buffer as it is (off after a power loss), then the + * input enable in a write of its own, the lock last. A locked pin is left alone. Written + * in one go, with the input enable among the rest, the pin reads 0 for good after the wake. + * That is established for the T6041 blocks; any other block gets its word whole, as before. + */ +static void apple_gpio_safe_reconfigure(struct apple_gpio_pinctrl *pctl, unsigned int pin, + u32 old, u32 cfg) +{ + u32 val = cfg & ~(REG_GPIOx_LOCK | REG_GPIOx_INPUT_ENABLE); + + if (!pctl->staged) { + regmap_write(pctl->map, REG_GPIO(pin), cfg); + return; + } + if (old & REG_GPIOx_LOCK) + return; + if (!(cfg & REG_GPIOx_INPUT_ENABLE)) { + regmap_write(pctl->map, REG_GPIO(pin), + old & ~(REG_GPIOx_LOCK | REG_GPIOx_INPUT_ENABLE)); + regmap_write(pctl->map, REG_GPIO(pin), val); + } else { + regmap_write(pctl->map, REG_GPIO(pin), (old & REG_GPIOx_INPUT_ENABLE) | val); + if (!(old & REG_GPIOx_INPUT_ENABLE)) + regmap_write(pctl->map, REG_GPIO(pin), cfg & ~REG_GPIOx_LOCK); + } + if (cfg & REG_GPIOx_LOCK) + regmap_write(pctl->map, REG_GPIO(pin), cfg); +} + /* * A firmware-driven suspend powers this block off, and neither the pin core nor * a consumer sets a pin up again. Slots that fault aren't readable in the regmap, @@ -673,15 +757,34 @@ static int apple_gpio_pinctrl_suspend_noirq(struct device *dev) struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); unsigned int i; - if (!pctl->saved || pm_suspend_target_state != PM_SUSPEND_MEM || - !pm_suspend_via_firmware()) + if (pm_suspend_target_state != PM_SUSPEND_MEM || !pm_suspend_via_firmware()) + return 0; + + bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); + /* + * A block that loses every pin: every pin as the hardware has it now, but the ones + * the device tree lists in gpio-reserved-ranges, which may not be written back. The + * pins of the panel's TCON bus, set up by the boot loader, are among them. + */ + pctl->saved_global_valid = false; + if (pctl->restore_all) { + pctl->saved_global = readl(pctl->base + REG_GLOBAL); + pctl->saved_global_valid = true; + if (pctl->cached) + regcache_cache_bypass(pctl->map, true); + for (i = 0; i < pctl->pinctrl_desc.npins; i++) + if (!test_bit(i, pctl->no_restore) && + !regmap_read(pctl->map, REG_GPIO(i), &pctl->saved[i])) + set_bit(i, pctl->saved_valid); + if (pctl->cached) + regcache_cache_bypass(pctl->map, false); return 0; + } /* * Only the pins this driver configured. The others belong to the firmware or another * processor, and writing one back after the wake raises an SError. */ - bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); for_each_set_bit(i, pctl->touched, pctl->pinctrl_desc.npins) if (!regmap_read(pctl->map, REG_GPIO(i), &pctl->saved[i])) set_bit(i, pctl->saved_valid); @@ -694,25 +797,84 @@ static int apple_gpio_pinctrl_resume_noirq(struct device *dev) struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); unsigned int i, val, restored = 0; - if (!pctl->saved || !pm_resume_via_firmware()) + if (!pm_resume_via_firmware()) return 0; + if (pctl->saved_global_valid) { + val = readl(pctl->base + REG_GLOBAL); + writel(pctl->saved_global, pctl->base + REG_GLOBAL); + dev_dbg(dev, "resume: global word %#x, put back to %#x\n", val, + pctl->saved_global); + pctl->saved_global_valid = false; + } + + /* The saved words came from the cache; the hardware is what to compare. */ + if (pctl->cached) + regcache_cache_bypass(pctl->map, true); for_each_set_bit(i, pctl->saved_valid, pctl->pinctrl_desc.npins) { - if (!regmap_read(pctl->map, REG_GPIO(i), &val) && val == pctl->saved[i]) + if (regmap_read(pctl->map, REG_GPIO(i), &val)) + val = 0; + else if (val == pctl->saved[i]) continue; - regmap_write(pctl->map, REG_GPIO(i), pctl->saved[i]); + apple_gpio_safe_reconfigure(pctl, i, val, pctl->saved[i]); restored++; } - dev_dbg(dev, "resume: %u of %u configured pins put back\n", restored, - bitmap_weight(pctl->saved_valid, pctl->pinctrl_desc.npins)); + if (pctl->cached) + regcache_cache_bypass(pctl->map, false); + if (restored) + dev_dbg(dev, "resume: %u of %u configured pins put back\n", restored, + bitmap_weight(pctl->saved_valid, pctl->pinctrl_desc.npins)); bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); return 0; } +/* + * A resume from disk is a cold boot: the block holds what the boot loader and + * the boot kernel set up, not the restored kernel's pins, while the regmap + * cache still holds the restored kernel's words. Neither an interrupt nor a + * consumer gets its pin back by itself: the IRQ core disables lazily and + * unmasks only an IRQ it recorded as masked, and regmap_update_bits() skips a + * write the cache already has. So an edge interrupt armed before the + * hibernation, such as the HDMI hotplug detect, stays off for good, and an + * output that its consumer sets to the level it had before is never driven. + * Write back, from the cache, every pin this driver set up that the block + * holds differently, in the order a pin takes its word. + * + * The early restore, as a PM domain does not wrap it (its restore_noirq ends + * in the driver's resume_noirq). An uncached block reads back only what the + * boot left, so this does nothing there. + */ +static int apple_gpio_pinctrl_restore_early(struct device *dev) +{ + struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); + unsigned int i, cfg, val, restored = 0; + + /* Uncached: no word of the restored kernel to write back. */ + if (!pctl->cached) + return 0; + + for_each_set_bit(i, pctl->touched, pctl->pinctrl_desc.npins) { + if (regmap_read(pctl->map, REG_GPIO(i), &cfg)) + continue; + /* Past the cache, which every write below leaves at @cfg again. */ + val = readl_relaxed(pctl->base + REG_GPIO(i)); + if (val == cfg) + continue; + apple_gpio_safe_reconfigure(pctl, i, val, cfg); + restored++; + } + if (restored) + dev_dbg(dev, "resume from disk: %u of %u configured pins put back\n", restored, + bitmap_weight(pctl->touched, pctl->pinctrl_desc.npins)); + + return 0; +} + static const struct dev_pm_ops apple_gpio_pinctrl_pm_ops = { NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_gpio_pinctrl_suspend_noirq, apple_gpio_pinctrl_resume_noirq) + .restore_early = pm_sleep_ptr(apple_gpio_pinctrl_restore_early), }; static struct platform_driver apple_gpio_pinctrl_driver = { diff --git a/drivers/pmdomain/apple/pmc-test.c b/drivers/pmdomain/apple/pmc-test.c index 4e7f19d47317a..5e0446c8fe21b 100644 --- a/drivers/pmdomain/apple/pmc-test.c +++ b/drivers/pmdomain/apple/pmc-test.c @@ -76,6 +76,23 @@ static void apple_pmc_test_agent_rails(struct kunit *test) rail < 2 ? BIT(1) : 0); } +/* After a resume from disk the enables are written again from the books. */ +static void apple_pmc_test_restore_after_reset(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + unsigned int rail; + + pmc->hw = &apple_pmc_hw_t6040; + KUNIT_ASSERT_EQ(test, apple_pmc_agent_enable(pmc, 1, true), 0); + for (rail = 0; rail < pmc->hw->rails; rail++) + writel(0, pmc->rails + pmc->hw->rail_enable + 4 * rail); + + KUNIT_ASSERT_EQ(test, apple_pmc_restore_state(pmc), 0); + for (rail = 0; rail < pmc->hw->rails; rail++) + KUNIT_EXPECT_EQ(test, apple_pmc_enable_read(pmc, rail), + rail < 2 ? BIT(1) : 0); +} + /* Lifting a DCS limit gives the AP's agent back its own vote, not 0. */ static void apple_pmc_test_dcs_limit_restores_ap_vote(struct kunit *test) { @@ -550,6 +567,7 @@ static struct kunit_case apple_pmc_test_cases[] = { KUNIT_CASE(apple_pmc_test_pending_floor), KUNIT_CASE(apple_pmc_test_agent_enable), KUNIT_CASE(apple_pmc_test_agent_rails), + KUNIT_CASE(apple_pmc_test_restore_after_reset), KUNIT_CASE(apple_pmc_test_dcs_limit_restores_ap_vote), KUNIT_CASE(apple_pmc_test_floor_aggregate), KUNIT_CASE(apple_pmc_test_floor_pending_timeout), diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 586e3f675db9e..2341ff1d0c9fa 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -1534,16 +1534,12 @@ static const struct apple_pmc_hw apple_pmc_hw_t6050 = { * A system sleep that reached the hardware may have taken the enables and floors * with it. Put back what the driver's books say: these are no-ops if they stayed. */ -static int apple_pmc_resume_early(struct device *dev) +static int apple_pmc_restore_state(struct apple_pmc *pmc) { - struct apple_pmc *pmc = dev_get_drvdata(dev); unsigned long flags; unsigned int rail; int ret = 0, err; - if (!pm_resume_via_firmware() || !pmc) - return 0; - raw_spin_lock_irqsave(&pmc->lock, flags); for (rail = 0; rail < pmc->hw->rails; rail++) { err = apple_pmc_enable_write(pmc, rail); @@ -1554,14 +1550,49 @@ static int apple_pmc_resume_early(struct device *dev) } } raw_spin_unlock_irqrestore(&pmc->lock, flags); + + return ret; +} + +static int apple_pmc_resume_early(struct device *dev) +{ + struct apple_pmc *pmc = dev_get_drvdata(dev); + int ret; + + if (!pm_resume_via_firmware() || !pmc) + return 0; + + ret = apple_pmc_restore_state(pmc); if (ret) dev_err(dev, "state not restored after sleep: %d\n", ret); return 0; } +/* + * The kernel restored from a hibernation image runs after a boot loader that + * started from a reset PMC: every agent is off on every rail again, so the + * floors of the agents this driver enabled (the CPU performance controller's + * DCS and fabric votes among them) are no longer honoured. Restore always. + */ +static int apple_pmc_restore_early(struct device *dev) +{ + struct apple_pmc *pmc = dev_get_drvdata(dev); + int ret; + + if (!pmc) + return 0; + + ret = apple_pmc_restore_state(pmc); + if (ret) + dev_err(dev, "state not restored after resume from disk: %d\n", ret); + + return 0; +} + static const struct dev_pm_ops apple_pmc_pm_ops = { - LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_pmc_resume_early) + .resume_early = pm_sleep_ptr(apple_pmc_resume_early), + .restore_early = pm_sleep_ptr(apple_pmc_restore_early), }; static const struct of_device_id apple_pmc_of_match[] = { diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index d6ac0c2cebd9f..ee7cf19d5816a 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -672,6 +672,46 @@ static int apple_pmgr_ps_resume_noirq(struct device *dev) return 0; } +/* + * After a resume from disk genpd still records the domains as they were when + * the image was taken, but the cold boot behind the resume left the hardware + * in the boot loader's state, and genpd powers on only the domains it recorded + * as off. A domain recorded on but found gated would make its consumers' + * restore touch an unpowered block. Bring each such domain up, parents first. + */ +static void apple_pmgr_ps_restore_hw(struct apple_pmgr_ps *ps, unsigned int depth) +{ + struct gpd_link *link; + u32 reg; + + if (depth > 16) + return; + list_for_each_entry(link, &ps->genpd.child_links, child_node) + if (link->parent->power_on == apple_pmgr_ps_power_on) + apple_pmgr_ps_restore_hw(genpd_to_apple_pmgr_ps(link->parent), depth + 1); + + if (ps->firmware_owned || ps->inherited_on || ps->sleep_driver_owned || + ps->genpd.status != GENPD_STATE_ON) + return; + if (regmap_read(ps->regmap, ps->offset, ®)) + return; + if (FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) == APPLE_PMGR_PS_ACTIVE) + return; + dev_dbg(ps->dev, "PS %s: on in the image, gated after the cold boot (0x%x): powering on\n", + ps->genpd.name, reg); + apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_ACTIVE, + !ps->no_auto_enable && !ps->no_auto_pm); +} + +static int apple_pmgr_ps_restore_noirq(struct device *dev) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + + if (ps) + apple_pmgr_ps_restore_hw(ps, 0); + return 0; +} + static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { struct apple_pmgr_ps *ps = rcdev_to_apple_pmgr_ps(rcdev); @@ -1383,7 +1423,13 @@ MODULE_DEVICE_TABLE(of, apple_pmgr_ps_of_match); static const struct dev_pm_ops apple_pmgr_ps_pm_ops = { LATE_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_late, apple_pmgr_ps_resume_early) - NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_noirq, apple_pmgr_ps_resume_noirq) + .suspend_noirq = pm_sleep_ptr(apple_pmgr_ps_suspend_noirq), + .freeze_noirq = pm_sleep_ptr(apple_pmgr_ps_suspend_noirq), + .poweroff_noirq = pm_sleep_ptr(apple_pmgr_ps_suspend_noirq), + /* Suspend-to-RAM resyncs after the sleep engine; a resume from disk after the cold boot. */ + .resume_noirq = pm_sleep_ptr(apple_pmgr_ps_resume_noirq), + .thaw_noirq = pm_sleep_ptr(apple_pmgr_ps_resume_noirq), + .restore_noirq = pm_sleep_ptr(apple_pmgr_ps_restore_noirq), }; static struct platform_driver apple_pmgr_ps_driver = { diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 7ce01a88bb069..055f6e42760a6 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -447,6 +447,10 @@ static void apple_pmp_report_ready_work(struct work_struct *work) unsigned int ms = rep->polls++ < PMP_REPORT_FAST_POLLS ? PMP_REPORT_POLL_MS : PMP_REPORT_POLL_SLOW_MS; + if (rep->polls == PMP_REPORT_FAST_POLLS + 1) + dev_warn(rep->dev, "PMP still stopped, requests %#llx held\n", + rep->shadow); + queue_delayed_work(system_freezable_power_efficient_wq, &rep->ready_work, msecs_to_jiffies(ms)); return; @@ -559,6 +563,90 @@ void apple_pmp_report_wake(struct device *pmp, bool lost_power) } EXPORT_SYMBOL_GPL(apple_pmp_report_wake); +/* + * Hold requests again for a PMP that stopped behind this driver's back: on + * a resume from hibernation the boot loader has cold-restarted the PMP + * (halted, its report region reset when it starts) while the restored image + * still has requests handed. Requests made from now on stay in @shadow and + * the ready work hands all of them over once the PMP runs, as at probe. + * Returns whether the requests are held now. Called with @rep->lock held. + */ +static bool apple_pmp_report_hold_locked(struct apple_pmp_report *rep) +{ + if (rep->defer) + return true; + if (!rep->offsets->defer_requests || apple_pmp_report_running(rep)) + return false; + rep->defer = true; + rep->polls = 0; + queue_delayed_work(system_freezable_power_efficient_wq, &rep->ready_work, 0); + return true; +} + +static bool apple_pmp_report_handed(struct apple_pmp_report *rep) +{ + unsigned long flags; + bool handed; + u64 want; + + spin_lock_irqsave(&rep->lock, flags); + handed = !rep->defer; + want = rep->shadow; + spin_unlock_irqrestore(&rep->lock, flags); + return handed && (readq(rep->base + rep->offsets->actual) & want) == want; +} + +/** + * apple_pmp_report_wait_handed() - wait until every held request is acknowledged + * @timeout_ms: upper bound of the wait + * + * For a consumer that restarts its own firmware after a resume from + * hibernation and must not run it before the PMP has taken the power state + * requests made for it (the GPU's pmp-gfx request). Sleeps. + * + * Return: 0 once every report runs with its requests acknowledged, or + * -ETIMEDOUT. + */ +int apple_pmp_report_wait_handed(unsigned int timeout_ms) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); + struct apple_pmp_report *rep; + bool done; + + for (;;) { + done = true; + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) + done &= apple_pmp_report_handed(rep); + rcu_read_unlock(); + if (done) + return 0; + if (time_after(jiffies, deadline)) + return -ETIMEDOUT; + msleep(10); + } +} +EXPORT_SYMBOL_GPL(apple_pmp_report_wait_handed); + +static int apple_pmp_report_restore_noirq(struct device *dev) +{ + struct apple_pmp_report *rep = dev_get_drvdata(dev); + unsigned long flags; + bool held; + u64 want; + + if (!rep) + return 0; + spin_lock_irqsave(&rep->lock, flags); + held = !rep->defer && apple_pmp_report_hold_locked(rep); + want = rep->shadow; + spin_unlock_irqrestore(&rep->lock, flags); + if (held) + dev_info(dev, "PMP stopped after restore: requests %#llx held until it runs\n", + want); + return 0; +} + /* The unique, enabled and acknowledged T6030 display request (DISP, reg 7). */ static struct device_node *apple_pmp_report_t6030_entry(struct device_node *report) { @@ -2269,8 +2357,10 @@ static int apple_pmp_report_resume(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_pmp_report_pm_ops, apple_pmp_report_suspend, - apple_pmp_report_resume); +static const struct dev_pm_ops apple_pmp_report_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_pmp_report_suspend, apple_pmp_report_resume) + .restore_noirq = apple_pmp_report_restore_noirq, +}; static int apple_pmp_report_probe(struct platform_device *pdev) { @@ -2555,7 +2645,8 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, rep->shadow |= bit_val; else rep->shadow &= ~bit_val; - ready = !rep->defer; + /* Never write a request to a stopped PMP (see the hold helper). */ + ready = !apple_pmp_report_hold_locked(rep); if (ready) { val = readq(rep->base + rep->offsets->tgt_read); val &= ~bit_val; diff --git a/drivers/power/reset/macsmc-reboot.c b/drivers/power/reset/macsmc-reboot.c index a2e1422e05bd8..de8161a96eabe 100644 --- a/drivers/power/reset/macsmc-reboot.c +++ b/drivers/power/reset/macsmc-reboot.c @@ -127,15 +127,33 @@ static int macsmc_prepare_atomic(struct sys_off_data *data) static int macsmc_power_off(struct sys_off_data *data) { struct macsmc_reboot *reboot = data->cb_data; + int ret, i; dev_info(reboot->dev, "Issuing power off (off1)\n"); - if (apple_smc_write_u32_atomic(reboot->smc, SMC_KEY(MBSE), SMC_KEY(off1)) < 0) { - dev_err(reboot->dev, "Failed to issue MBSE = off1 (power_off)\n"); - } else { + /* + * This is the only power-off path when Linux runs at EL2 without a + * monitor, and where a PSCI SYSTEM_OFF follows, a monitor without a + * power-off path resets the machine instead. So a slow or refused + * off1 must not fall through after only 100 ms: retry a refused + * command and give an accepted one time to take effect, and say on + * the console which of the two happened. + */ + for (i = 0; i < 3; i++) { + ret = apple_smc_write_u32_atomic(reboot->smc, SMC_KEY(MBSE), SMC_KEY(off1)); + if (ret >= 0) + break; + dev_warn(reboot->dev, "MBSE = off1 refused (%d), attempt %d\n", ret, i + 1); mdelay(100); - WARN_ONCE(1, "Unable to power off system\n"); } + if (ret < 0) { + dev_err(reboot->dev, "Failed to issue MBSE = off1 (power_off)\n"); + return NOTIFY_OK; + } + dev_dbg(reboot->dev, "off1 accepted, waiting for the power to drop\n"); + for (i = 0; i < 50; i++) + mdelay(100); + WARN_ONCE(1, "Unable to power off system: off1 accepted, power still on after 5 s\n"); return NOTIFY_OK; } diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index cd46c9b9a10b1..2e07c1a393371 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -432,17 +432,59 @@ static int apple_pmu_rtc_prepare(struct device *dev) if (rtc->io_failed) return -EBUSY; /* + * Suspend to idle keeps the PMU powered and the alarm wakes the system. * A firmware-driven suspend programs the wake sources itself, and the - * alarm stays armed in the PMU, which stays powered. Only a sleep state - * without either has no way to bring the alarm back. + * alarm stays armed in the PMU, which stays powered. Hibernation (the + * target state stays PM_SUSPEND_ON) is accepted as well: the restore + * below has the alarm set again. Only a sleep state without any of + * these has no way to bring the alarm back. */ - if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && !pm_suspend_via_firmware()) + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && + pm_suspend_target_state != PM_SUSPEND_ON && !pm_suspend_via_firmware()) return -EBUSY; return 0; } +/* + * Only a hibernation that powers off through platform hibernation ops gets + * here; a plain power-off or reboot after the image leaves the alarm as it is. + * Whether an armed PMU alarm powers the machine on again after the + * hibernation power-off is not established, so do not leave one armed. + */ +static int apple_pmu_rtc_poweroff(struct device *dev) +{ + struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); + int error; + + mutex_lock(&rtc->transaction); + error = apple_pmu_rtc_stop_alarm(rtc); + mutex_unlock(&rtc->transaction); + if (error) + dev_warn(dev, "could not stop the alarm before power-off: %d\n", error); + return 0; +} + +/* + * A resume from disk went through a boot. The firmware and the kernel that + * loaded the image had the PMU in between: either may have stopped the alarm + * or set the comparator anew, and the alarm's time may have passed with the + * machine off, its event acknowledged by one of them. The RTC core still + * counts the alarm as set. Have it go through its timers: an alarm that is due + * is reported, a later one is written to the PMU again, and with none set the + * PMU's alarm is stopped. + */ +static int apple_pmu_rtc_restore(struct device *dev) +{ + struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); + + rtc_update_irq(rtc->clock, 1, RTC_IRQF | RTC_AF); + return 0; +} + static const struct dev_pm_ops apple_pmu_rtc_pm = { .prepare = apple_pmu_rtc_prepare, + .poweroff = apple_pmu_rtc_poweroff, + .restore = apple_pmu_rtc_restore, }; static void apple_pmu_rtc_release(void *context) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 9225b054e7cde..ef61ebcc97f61 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -98,6 +98,7 @@ config APPLE_SOC_SLEEP depends on ARM64 && (ARCH_APPLE || COMPILE_TEST) depends on HOTPLUG_CPU && SUSPEND select APPLE_TUNABLE + select MFD_SYSCON if MFD_MACSMC=y help Powers the secondary CPUs off and suspends the system to RAM on an Apple SoC whose kernel runs directly at EL2, with no monitor to ask. @@ -105,8 +106,25 @@ config APPLE_SOC_SLEEP /chosen/aurora,s2r-mailbox. Only SoCs whose sleep has been measured and tested are supported, - each by its match data: today T8140 and T8132. On every other SoC - the driver doesn't bind, and s2idle stays the only system sleep. + each by its match data: today T8140, T8132, and with MFD_MACSMC + built in the T6040. On every other SoC the driver doesn't bind, and + s2idle stays the only system sleep. + + The T6040 (M4 Pro) part has its own sequences and refuses first, on + the boards whose device tree enables its sleep controller: its + suspend handler lists every blocker in the live state (EL2, the boot + object's return path, the CPUs that can park, the display power + states that have to be off before a sleep). Unless + t6040_sleep.allow_entry=1 is given and no blocker remains, a request + for suspend-to-RAM prints the blockers and enters nothing, and + suspend-to-idle stays the default sleep. With it the SoC is powered + down to suspend-to-RAM and comes back through the boot object; the + AON watchdog is the net over the whole cycle + (t6040_sleep.sleep_wd_secs at the power-down itself, + t6040_sleep.wd2_secs over a dry run). With t6040_sleep.dry_run=1 + every driver's firmware-driven sleep step that can be taken back + runs, the CPUs park and are held, and the sleep is refused at the + commit point and resumed warm. config APPLE_SOC_SLEEP_KUNIT_TEST bool "KUnit tests for the Apple SoC sleep match data" if !KUNIT_ALL_TESTS @@ -320,6 +338,19 @@ config APPLE_CLPC Say 'y' or 'm' here if you have an Apple SoC with a PMC. +config APPLE_AURA_HIBERNATE + def_bool y + depends on ARCH_APPLE && HIBERNATION && OF_EARLY_FLATTREE + help + Under aura, the boot loader and the firmware keep memory alive + below Linux's allocations, and on Apple M4 Pro boots the boot + firmware places it at a different address every time. With + aura's fixed layout (/chosen/aura,hibernate-layout) this keeps + every reserved range out of the hibernation image; without it, + hibernation and resume are refused instead of overwriting the + running boot loader. The refusal is part of the hibernation + support of these machines, so the option cannot be turned off. + config APPLE_DISPLAY_GATE bool "Apple internal display enable gate" depends on ARCH_APPLE && OF_DYNAMIC && OF_OVERLAY diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 5065bb7a1324a..6f3f35604eb95 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -14,10 +14,14 @@ obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o obj-$(CONFIG_APPLE_CLUSTER_PWRGATE) += apple-cluster-pwrgate.o obj-$(CONFIG_APPLE_SOC_SLEEP) += apple-sleep.o -apple-sleep-y = sleep.o +apple-sleep-y = sleep.o sleep-registry.o +# The T6040 sleep calls the SMC driver directly +ifeq ($(CONFIG_MFD_MACSMC),y) +apple-sleep-y += sleep-t6040.o +endif obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o -apple-rtkit-y = rtkit.o rtkit-crashlog.o +apple-rtkit-y = rtkit.o rtkit-crashlog.o rtkit-hib-handoff.o obj-$(CONFIG_APPLE_RTKIT_HELPER) += apple-rtkit-helper.o apple-rtkit-helper-y = rtkit-helper.o @@ -42,6 +46,8 @@ DTC_FLAGS_t6030-gate-test := -Wno-interrupts_property -Wno-chosen_node_is_root \ DTC_FLAGS_t8122-gate-test := -Wno-interrupts_property -Wno-chosen_node_is_root DTC_FLAGS_t6041-gate-test := -Wno-unit_address_vs_reg -Wno-avoid_unnecessary_addr_size +obj-$(CONFIG_APPLE_AURA_HIBERNATE) += aura-hibernate.o + obj-$(CONFIG_APPLE_TUNABLE) += apple-tunable.o apple-tunable-y = tunable.o diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 95c9c8bcdea70..c66664e279c95 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -11,7 +11,8 @@ use core::{ marker::Unpin, mem, ptr::{self, NonNull}, - slice, // + slice, + sync::atomic::{AtomicI32, AtomicU64, Ordering}, // }; use kernel::{ @@ -1242,6 +1243,33 @@ const RETIRED_DRIVER_OVERRIDE: &CStr = c_str!("apple-aop-retired"); /// before a reboot. static RETIRED: Atomic = Atomic::new(false); +/// Set by unbind when the ASC CPU run bit reads 0: the co-processor was +/// halted underneath this instance, as the boot loader leaves it after a +/// resume from disk (a cold boot; the restored kernel still holds the session +/// of the instance that probed before the image). A halted CPU neither +/// answers nor DMAs, so the teardown sends nothing and frees everything, and +/// the next probe starts the firmware the boot loader staged, as at boot. +static HALTED_AT_UNBIND: Atomic = Atomic::new(false); + +/// Set by unbind with [`HALTED_AT_UNBIND`], cleared by the next successful +/// probe: the next probe starts firmware staged on a boot that is not the one +/// this kernel was started on, so its DRAM segments are taken from that boot +/// (see [`AopData::remap_fw_regions_cold`]). +static COLD_REPROBE: Atomic = Atomic::new(false); + +/// Set with [`COLD_REPROBE`] when [`RETIRED`] was already set at that halted +/// unbind: the failed shutdown predates the restaging, so the cold probe may +/// bring the co-processor up again. +static RETIRED_BEFORE_HALT: Atomic = Atomic::new(false); + +/// The firmware regions mapped in the default domain now (phys, IOVA, +/// size): -1 until the first probe takes them from the DT; a cold probe +/// replaces them with the resuming boot's. +const FW_SET_MAX: usize = 8; +static FW_CUR_LEN: AtomicI32 = AtomicI32::new(-1); +static FW_CUR: [[AtomicU64; 3]; FW_SET_MAX] = + [const { [const { AtomicU64::new(0) }; 3] }; FW_SET_MAX]; + impl ChildDevice { /// Keeps the child from binding to any driver again; it stays registered. fn retire(&self) { @@ -1976,26 +2004,9 @@ impl AopData { Ok((phys, iova, size)) } - /// Checks the firmware segments the AOP runs from DRAM (T6040). The - /// bootloader reserves them and lists them in `memory-region`, and each - /// one's `iommu-addresses` gives the IOVA the firmware uses for it. The - /// IOMMU core maps them in the default domain when the device is added - /// (one to one when the IOVA is the physical address, translated - /// otherwise), so they are only checked here. Every page has to be mapped - /// where the DT says, the external code (`__ETEXT`) at `ec0p` and any - /// other segment (`__OS_LOG`) at its physical address. - fn check_fw_regions(dev: &device::Device, ec0p: u64) -> Result<()> { - // SAFETY: `dev` is a live device being probed. - let np = unsafe { (*dev.as_raw()).of_node }; - if np.is_null() { - return Err(ENODEV); - } - // SAFETY: `np` is the device's node, valid while it is bound. - let count = unsafe { bindings::of_reserved_mem_region_count(np) }; - if count <= 0 { - dev_err!(dev, "No firmware memory-region from the bootloader"); - return Err(ENOENT); - } + /// The device's translated IOMMU domain, which the firmware segments are + /// mapped in. + fn fw_domain(dev: &device::Device) -> Result<*mut bindings::iommu_domain> { // SAFETY: `dev` is a live device being probed. let domain = unsafe { bindings::iommu_get_domain_for_dev(dev.as_raw()) }; let ty = if domain.is_null() { @@ -2008,59 +2019,331 @@ impl AopData { dev_err!(dev, "No translated IOMMU domain for the firmware"); return Err(ENODEV); } - let mut etext = false; - for idx in 0..count { - let (phys, iova, size) = Self::fw_region(np, idx).inspect_err(|e| { - dev_err!(dev, "Invalid firmware memory-region {} ({:?})", idx, e); - })?; - if phys % FW_REGION_PAGE != 0 - || size % FW_REGION_PAGE != 0 - || size > FW_REGION_MAX_SIZE - || iova % FW_REGION_PAGE != 0 - || iova.checked_add(size).is_none() - || phys.checked_add(size).is_none() - { - dev_err!(dev, "Invalid firmware memory-region {}", idx); + Ok(domain) + } + + /// The device's node and its number of firmware memory-regions. + fn fw_node(dev: &device::Device) -> Result<(*mut bindings::device_node, c_int)> { + // SAFETY: `dev` is a live device being probed. + let np = unsafe { (*dev.as_raw()).of_node }; + if np.is_null() { + return Err(ENODEV); + } + // SAFETY: `np` is the device's node, valid while it is bound. + let count = unsafe { bindings::of_reserved_mem_region_count(np) }; + if count <= 0 { + dev_err!(dev, "No firmware memory-region from the bootloader"); + return Err(ENOENT); + } + Ok((np, count)) + } + + /// The layout rules of one firmware memory-region: 16K pages, at most + /// [`FW_REGION_MAX_SIZE`], the IOVA either `ec0p` (at most one region, + /// counted in `etext`) or the physical address. + fn check_fw_region_layout( + dev: &device::Device, + ec0p: u64, + idx: c_int, + (phys, iova, size): (u64, u64, u64), + etext: &mut bool, + ) -> Result<()> { + if phys % FW_REGION_PAGE != 0 + || size % FW_REGION_PAGE != 0 + || size > FW_REGION_MAX_SIZE + || iova % FW_REGION_PAGE != 0 + || iova.checked_add(size).is_none() + || phys.checked_add(size).is_none() + { + dev_err!(dev, "Invalid firmware memory-region {}", idx); + return Err(EINVAL); + } + if iova == ec0p { + if *etext { + dev_err!(dev, "Two firmware memory-regions at EC0p {:#x}", ec0p); return Err(EINVAL); } - if iova == ec0p { - if etext { - dev_err!(dev, "Two firmware memory-regions at EC0p {:#x}", ec0p); - return Err(EINVAL); - } - etext = true; - } else if iova != phys { + *etext = true; + } else if iova != phys { + dev_err!( + dev, + "Firmware memory-region {} at IOVA {:#x}, neither EC0p {:#x} nor its physical address", + idx, + iova, + ec0p + ); + return Err(EINVAL); + } + Ok(()) + } + + /// Every page of a firmware memory-region has to be mapped at its IOVA. + fn check_fw_region_mapped( + dev: &device::Device, + domain: *mut bindings::iommu_domain, + idx: c_int, + (phys, iova, size): (u64, u64, u64), + ) -> Result<()> { + let mut off = 0; + while off < size { + // SAFETY: `domain` is the device's domain, live while it is + // bound. + let back = unsafe { bindings::iommu_iova_to_phys(domain, iova + off) }; + if back != phys + off { dev_err!( dev, - "Firmware memory-region {} at IOVA {:#x}, neither EC0p {:#x} nor its physical address", + "Firmware memory-region {} is not mapped at IOVA {:#x} (region phys {:#x} iova {:#x} size {:#x}, domain gives {:#x})", idx, + iova + off, + phys, iova, - ec0p + size, + back ); - return Err(EINVAL); + return Err(EIO); + } + off += FW_REGION_PAGE; + } + Ok(()) + } + + /// The firmware regions the default domain maps now: the DT's until a + /// resume from disk replaced them with those of the boot that resumed. + fn fw_set_load(dev: &device::Device) -> Result> { + let mut set = KVec::new(); + let len = FW_CUR_LEN.load(Ordering::Acquire); + if len >= 0 { + for r in FW_CUR.iter().take(len as usize) { + set.push( + ( + r[0].load(Ordering::Relaxed), + r[1].load(Ordering::Relaxed), + r[2].load(Ordering::Relaxed), + ), + GFP_KERNEL, + )?; + } + return Ok(set); + } + let (np, count) = Self::fw_node(dev)?; + for idx in 0..count { + let region = Self::fw_region(np, idx).inspect_err(|e| { + dev_err!(dev, "Invalid firmware memory-region {} ({:?})", idx, e); + })?; + set.push(region, GFP_KERNEL)?; + } + Ok(set) + } + + fn fw_set_store(set: &[(u64, u64, u64)]) -> Result<()> { + if set.len() > FW_SET_MAX { + return Err(E2BIG); + } + for (slot, r) in FW_CUR.iter().zip(set.iter()) { + slot[0].store(r.0, Ordering::Relaxed); + slot[1].store(r.1, Ordering::Relaxed); + slot[2].store(r.2, Ordering::Relaxed); + } + FW_CUR_LEN.store(set.len() as i32, Ordering::Release); + Ok(()) + } + + /// The layout rules and the mappings of a whole set: every region valid, + /// exactly one at `ec0p`, every page mapped. + fn check_fw_set( + dev: &device::Device, + domain: *mut bindings::iommu_domain, + ec0p: u64, + set: &[(u64, u64, u64)], + ) -> Result<()> { + let mut etext = false; + for (idx, r) in set.iter().enumerate() { + Self::check_fw_region_layout(dev, ec0p, idx as c_int, *r, &mut etext)?; + Self::check_fw_region_mapped(dev, domain, idx as c_int, *r)?; + } + if !etext { + dev_err!(dev, "No firmware memory-region at EC0p {:#x}", ec0p); + return Err(ENOENT); + } + Ok(()) + } + + /// Checks the firmware segments the AOP runs from DRAM (T6040). The + /// bootloader reserves them and lists them in `memory-region`, and each + /// one's `iommu-addresses` gives the IOVA the firmware uses for it. The + /// IOMMU core maps them in the default domain when the device is added + /// (one to one when the IOVA is the physical address, translated + /// otherwise), so they are only checked here. Every page has to be mapped + /// where the DT says, the external code (`__ETEXT`) at `ec0p` and any + /// other segment (`__OS_LOG`) at its physical address. After a resume + /// from disk the set checked is the one [`Self::remap_fw_regions_cold`] + /// mapped. + fn check_fw_regions(dev: &device::Device, ec0p: u64) -> Result<()> { + let set = Self::fw_set_load(dev)?; + let domain = Self::fw_domain(dev)?; + Self::check_fw_set(dev, domain, ec0p, &set)?; + Self::fw_set_store(&set) + } + + /// The firmware segments after a resume from disk (T6040 under aura). + /// + /// The resume is a cold boot: the boot firmware staged the AOP firmware anew and + /// placed some of its DRAM segments elsewhere than on the boot the + /// current mappings come from (the `__OS_LOG` moves on every boot, + /// `__ETEXT` has stayed in place so far). This boot's regions come from + /// the device tree aura handed over this time, at its fixed address; the + /// previous set's mappings that this boot does not have are removed, this + /// boot's are mapped (IOVA as the device tree says), checked as at boot, + /// and become the current set. Nothing is copied: the old physical ranges + /// may hold other firmware on this boot. Without aura's device tree slot + /// the current set is only checked, as before. + #[cfg(CONFIG_APPLE_AURA_HIBERNATE)] + fn remap_fw_regions_cold(dev: &device::Device, ec0p: u64) -> Result<()> { + let (np, _) = Self::fw_node(dev)?; + let prev = Self::fw_set_load(dev)?; + let domain = Self::fw_domain(dev)?; + let mut raw = [bindings::aura_fw_region { + phys: 0, + iova: 0, + size: 0, + }; FW_SET_MAX]; + // SAFETY: `np` is the device's node and `raw` has room for + // `FW_SET_MAX` entries. + let n = unsafe { + bindings::aura_hibernate_boot_fw_regions(np, raw.as_mut_ptr(), FW_SET_MAX as u32) + }; + if n == -(bindings::ENODEV as c_int) { + dev_warn!( + dev, + "resume from disk: no device tree of this boot from aura; checking the current firmware regions" + ); + return Self::check_fw_regions(dev, ec0p); + } + if n < 0 { + dev_err!( + dev, + "This boot's firmware memory-regions are not available ({}); the firmware cannot be mapped", + n + ); + return Err(Error::from_errno(n)); + } + let boot: KVec<(u64, u64, u64)> = { + let mut v = KVec::new(); + for r in &raw[..n as usize] { + v.push((r.phys, r.iova, r.size), GFP_KERNEL)?; + } + v + }; + let mut etext = false; + for (idx, r) in boot.iter().enumerate() { + Self::check_fw_region_layout(dev, ec0p, idx as c_int, *r, &mut etext)?; + } + if !etext { + dev_err!(dev, "No firmware memory-region at EC0p {:#x} on this boot", ec0p); + return Err(ENOENT); + } + // Mapped like the IOMMU core maps a direct region of this device. + // SAFETY: `np` is a valid node. + let coherent = unsafe { bindings::of_dma_is_coherent(np) }; + let mut prot = (bindings::IOMMU_READ | bindings::IOMMU_WRITE) as c_int; + if coherent { + prot |= bindings::IOMMU_CACHE as c_int; + } + + // The previous set's regions that this boot moved: unmap only pages + // that still map the old physical range, never anything else at that + // IOVA. Repeating it after a partial failure is harmless. + for (idx, &(phys, iova, size)) in prev.iter().enumerate() { + if boot.iter().any(|r| *r == (phys, iova, size)) { + continue; } let mut off = 0; + let mut unmapped = 0; while off < size { // SAFETY: `domain` is the device's domain, live while it is // bound. let back = unsafe { bindings::iommu_iova_to_phys(domain, iova + off) }; - if back != phys + off { - dev_err!( - dev, - "Firmware memory-region {} is not mapped at IOVA {:#x}", - idx, - iova + off - ); - return Err(EIO); + if back == phys + off { + // SAFETY: As above; the page was mapped for this region. + let done = unsafe { + bindings::iommu_unmap(domain, (iova + off) as usize, FW_REGION_PAGE as usize) + }; + if done != FW_REGION_PAGE as usize { + dev_err!(dev, "Could not unmap IOVA {:#x} of the old boot", iova + off); + return Err(EIO); + } + unmapped += 1; } off += FW_REGION_PAGE; } + dev_info!( + dev, + "resume from disk: old firmware region {} [{:#x}, +{:#x}) at IOVA {:#x} dropped ({} pages unmapped)", + idx, + phys, + size, + iova, + unmapped + ); } - if !etext { - dev_err!(dev, "No firmware memory-region at EC0p {:#x}", ec0p); - return Err(ENOENT); + + // This boot's regions: map what is missing, refuse what is taken. + for (idx, &(phys, iova, size)) in boot.iter().enumerate() { + let mut off = 0; + let mut mapped = 0; + while off < size { + // SAFETY: `domain` is the device's domain, live while it is + // bound. + let back = unsafe { bindings::iommu_iova_to_phys(domain, iova + off) }; + if back != phys + off { + if back != 0 { + dev_err!( + dev, + "IOVA {:#x} for this boot's firmware region {} is in use ({:#x})", + iova + off, + idx, + back + ); + return Err(EBUSY); + } + // SAFETY: As above; the IOVA page is free and the physical + // page is this boot's firmware segment, which Linux does + // not own (checked by the helper). + let err = unsafe { + bindings::iommu_map( + domain, + (iova + off) as usize, + phys + off, + FW_REGION_PAGE as usize, + prot, + bindings::GFP_KERNEL, + ) + }; + if err != 0 { + dev_err!(dev, "Could not map IOVA {:#x} ({})", iova + off, err); + return Err(Error::from_errno(err)); + } + mapped += 1; + } + off += FW_REGION_PAGE; + } + dev_info!( + dev, + "resume from disk: firmware region {} [{:#x}, +{:#x}) at IOVA {:#x} ({} pages mapped)", + idx, + phys, + size, + iova, + mapped + ); } - Ok(()) + Self::check_fw_set(dev, domain, ec0p, &boot)?; + Self::fw_set_store(&boot) + } + + #[cfg(not(CONFIG_APPLE_AURA_HIBERNATE))] + fn remap_fw_regions_cold(dev: &device::Device, ec0p: u64) -> Result<()> { + Self::check_fw_regions(dev, ec0p) } /// Leaks every allocation the firmware may still access. Called when the @@ -2287,6 +2570,16 @@ impl AOP for AopData { // No registration is queued after this point, so once the queue is // empty the list of children is complete. self.registration_queue.drain(); + let halted = HALTED_AT_UNBIND.xchg(false, Acquire) && self.cpu_started.load(Acquire); + if halted { + // Nobody answers: the children's unbind calls fail at once + // instead of each waiting for a reply. + self.transport_closing.store(true, Release); + dev_info!( + self.dev, + "co-processor halted underneath (resume from disk): releasing the old session without a shutdown" + ); + } // Unbind the children while the transport still works, so that their // unbind can talk to their services. Then drop any listener a child // left behind: no report may reach a driver that is going away. @@ -2297,8 +2590,10 @@ impl AOP for AopData { self.hid_listeners.lock().clear(); self.report_listeners.lock().clear(); self.transport_closing.store(true, Release); - if let Err(e) = self.stop() { - dev_err!(self.dev, "Failed to stop AOP {:?}", e); + if !halted { + if let Err(e) = self.stop() { + dev_err!(self.dev, "Failed to stop AOP {:?}", e); + } } // Take the handle out of the shared state before dropping it: the // drop waits for the RTKit receive worker, which takes the same lock. @@ -2306,7 +2601,7 @@ impl AOP for AopData { let rtkit = self.rtkit.lock().take(); let mut quiesced = true; if let Some(mut rtkit) = rtkit { - if self.quiesce_on_unbind && self.cpu_started.load(Acquire) { + if self.quiesce_on_unbind && self.cpu_started.load(Acquire) && !halted { // The co-processor DMAs into the shared buffers until it has // acknowledged the shutdown. If it does not, nothing it may // still write to can be freed. @@ -3069,13 +3364,122 @@ unsafe extern "C" fn aop_suspend_late(dev: *mut bindings::device) -> c_int { /// Called by the PM core with a bound AOP device. unsafe extern "C" fn aop_resume(dev: *mut bindings::device) -> c_int { // SAFETY: by this function's contract. - unsafe { aop_from_dev(dev) }.sleep_undo(); + let data = unsafe { aop_from_dev(dev) }; + // A sleep that powered the SoC down leaves the co-processor's run bit clear: its session is + // gone and nothing restarted it, so the warm wake's stop/start notices would each time out on + // a dead firmware (bounded by the RTKit handshake timeout, but seconds of stall) and the AOP + // would stay down until a reboot. Take it over the cold way instead, exactly as a resume from + // disk does. A co-processor that kept running through the sleep, as on T8140, has its run + // bit set, so this is a no-op there and the warm wake runs unchanged. + // SAFETY: only reads a flag. + if unsafe { bindings::pm_resume_via_firmware() } { + // SAFETY: by this function's contract. + if unsafe { aop_cold_lost(dev, data) } { + return 0; + } + } + data.sleep_undo(); + 0 +} + +/// A co-processor whose run bit is clear lost its power for this sleep. Close the transport so no +/// client waits on the dead firmware and queue the unbind and probe that start it again once tasks +/// thaw, as [`aop_restore`] does after a resume from disk. Returns whether it took the device over; +/// a co-processor still running is left to the warm wake. +/// +/// # Safety +/// +/// Called by a PM core resume callback with a bound AOP device; `data` is its drvdata. +unsafe fn aop_cold_lost(dev: *mut bindings::device, data: &AopData) -> bool { + if !data.cpu_started.load(Acquire) || data.removing.load(Acquire) { + return false; + } + // SAFETY: the PM core runs this callback on a bound device, under its device lock. + let bound = unsafe { device::Device::::from_raw(dev) }; + let Ok(pdev) = <&platform::Device>::try_from(bound) else { + return false; + }; + if !asc_cpu_halted(pdev) { + return false; + } + data.transport_closing.store(true, Release); + match restart_after_thaw(dev) { + Ok(()) => dev_info!( + data.dev, + "co-processor halted (power lost for the sleep), restarting it once tasks are thawed\n" + ), + Err(e) => dev_warn!( + data.dev, + "co-processor halted (power lost for the sleep), calls refused; restart not queued ({:?})\n", + e + ), + } + true +} + +/// Queues the unbind and probe that start a halted co-processor again. +#[cfg(CONFIG_APPLE_AURA_HIBERNATE)] +fn restart_after_thaw(dev: *mut bindings::device) -> Result<()> { + // SAFETY: `dev` is a live device; the helper holds a reference until its work ran. + to_result(unsafe { bindings::aura_hibernate_reprobe_after_thaw(dev) }) +} + +/// Without aura's hibernation glue nothing restarts it here: user space re-probes it. +#[cfg(not(CONFIG_APPLE_AURA_HIBERNATE))] +fn restart_after_thaw(_dev: *mut bindings::device) -> Result<()> { + Err(EOPNOTSUPP) +} + +/// A resume from disk is a cold boot: the boot loader staged the co-processor anew and left it +/// halted, and the boot kernel did not start it. The session this kernel holds is gone, but +/// nothing tells its clients: every call they make waits for a reply that never comes, and the +/// mailbox keeps the doorbells. The firmware the next probe starts would take them as its own +/// first messages while its endpoints come up, which it does not survive. So the transport +/// closes here, before any client runs, and the device is probed again once the tasks are +/// thawed: the unbind finds the run bit clear and drops the old session without a shutdown, +/// the probe starts the firmware as at boot. An aborted hibernation and a co-processor that +/// kept running are left as they are. +/// +/// # Safety +/// +/// Called by the PM core with a bound AOP device. +unsafe extern "C" fn aop_restore(dev: *mut bindings::device) -> c_int { + // SAFETY: by this function's contract. + let data = unsafe { aop_from_dev(dev) }; + if !data.cpu_started.load(Acquire) || data.removing.load(Acquire) { + return 0; + } + // SAFETY: the PM core runs this callback on a bound device, under its device lock. + let bound = unsafe { device::Device::::from_raw(dev) }; + let Ok(pdev) = <&platform::Device>::try_from(bound) else { + return 0; + }; + if !asc_cpu_halted(pdev) { + dev_info!( + data.dev, + "resume from disk: the co-processor kept running, its session is left as is\n" + ); + return 0; + } + data.transport_closing.store(true, Release); + match restart_after_thaw(dev) { + Ok(()) => dev_info!( + data.dev, + "resume from disk: co-processor halted (cold boot), restarting it once tasks are thawed\n" + ), + Err(e) => dev_warn!( + data.dev, + "resume from disk: co-processor halted (cold boot), calls refused; restart not queued ({:?})\n", + e + ), + } 0 } static AOP_PM_OPS: bindings::dev_pm_ops = bindings::dev_pm_ops { suspend_late: Some(aop_suspend_late), resume: Some(aop_resume), + restore: Some(aop_restore), // SAFETY: all-zero is a valid `dev_pm_ops`: no callbacks. ..unsafe { core::mem::zeroed() } }; @@ -3112,7 +3516,15 @@ impl platform::Driver for AopDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { let cfg = info.ok_or(ENODEV)?; - if RETIRED.load(Acquire) { + let cold = COLD_REPROBE.load(Acquire); + if cold && RETIRED.load(Acquire) && RETIRED_BEFORE_HALT.load(Acquire) { + // The boot loader halted and restaged the co-processor since the + // shutdown failed: nothing runs on the retained buffers anymore. + dev_warn!( + pdev.as_ref(), + "an earlier instance could not be shut down, but the co-processor was restaged since (resume from disk)" + ); + } else if RETIRED.load(Acquire) { dev_err!( pdev.as_ref(), "an earlier instance could not be shut down; reboot before probing again" @@ -3128,7 +3540,19 @@ impl platform::Driver for AopDriver { let asc_mmio = asc_mmio.as_view().relaxed(); if cfg.fw_regions { // Before anything is written to the AOP: no mapping, no start. - AopData::check_fw_regions(pdev.as_ref(), cfg.ec0p)?; + if cold { + // Remapping is only safe while the firmware does not run. + if asc_mmio.read(ASC_CPU_CONTROL).cpu_run() != 0 { + dev_err!( + pdev.as_ref(), + "resume from disk: the co-processor runs, its firmware regions are left alone" + ); + return Err(EBUSY); + } + AopData::remap_fw_regions_cold(pdev.as_ref(), cfg.ec0p)?; + } else { + AopData::check_fw_regions(pdev.as_ref(), cfg.ec0p)?; + } } let data = AopData::new(pdev, cfg)?; // Whatever fails below leaves the AOP running and its children @@ -3161,6 +3585,14 @@ impl platform::Driver for AopDriver { } asc_mmio.update(ASC_CPU_CONTROL, |r| r.with_const_cpu_run::<1>()); data.cpu_started.store(true, Release); + if cold { + // The firmware runs from here on: a later unbind sees it running + // and the next probe is an ordinary one, whatever happens below. + COLD_REPROBE.store(false, Release); + if RETIRED_BEFORE_HALT.xchg(false, Acquire) { + RETIRED.store(false, Release); + } + } let started = if cfg.setup_port { // The RTKit handshake up to AP power on comes first, then the // setup port's own power state and endpoint buffers, and only @@ -3198,13 +3630,33 @@ impl platform::Driver for AopDriver { Ok(Self(data)) } - fn unbind(_dev: &platform::Device>, this: Pin<&Self>) { + fn unbind(dev: &platform::Device>, this: Pin<&Self>) { + let halted = asc_cpu_halted(dev); + HALTED_AT_UNBIND.store(halted, Release); + if halted { + RETIRED_BEFORE_HALT.store(RETIRED.load(Acquire), Release); + COLD_REPROBE.store(true, Release); + } // The device is still bound here, which the DMA allocations the // teardown frees require. Drop only repeats it as a no-op. this.0.remove(); } } +/// Reads the ASC CPU run bit. Probe sets it; only the boot loader clears it +/// (the co-processor is staged halted at every boot, including the one a +/// resume from disk starts with). Any failure to read it counts as running, +/// which keeps the full shutdown. +fn asc_cpu_halted(pdev: &platform::Device) -> bool { + let Some(req) = pdev.io_request_by_index(1) else { + return false; + }; + let Ok(asc) = req.iomap_sized::() else { + return false; + }; + asc.as_view().relaxed().read(ASC_CPU_CONTROL).cpu_run() == 0 +} + impl Drop for AopDriver { fn drop(&mut self) { self.0.remove(); diff --git a/drivers/soc/apple/aura-hibernate.c b/drivers/soc/apple/aura-hibernate.c new file mode 100644 index 0000000000000..8c50b3cd33dba --- /dev/null +++ b/drivers/soc/apple/aura-hibernate.c @@ -0,0 +1,546 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Hibernation on Apple Silicon under an aura boot object. + * + * A hibernation image restores every saved page to the frame it was taken from, and swsusp saves + * reserved pages as long as they are mapped. Under aura the memory below Linux's own allocations + * holds what the boot loader keeps alive across the hand-off: aura itself (resident at EL2), its + * heap with the firmware buffers it handed over, the device tree, the initramfs, and the ranges + * the boot firmware placed for the co-processors. Restoring old contents there would overwrite + * the running boot loader and the live firmware state of the resuming boot. + * + * aura marks a boot whose layout is the same on every boot with /chosen/aura,hibernate-layout + * (start, end of the range it keeps from Linux). On such a boot every /memreserve/ range and every + * non-reusable /reserved-memory range is registered as nosave, except the kernel image itself. + * Without the property, hibernation and resume are refused, since the reserved ranges move. + * + * The boot firmware still places some firmware segments anew on every boot (the AOP __OS_LOG), + * inside the range aura keeps from Linux. A kernel restored from an image knows the device tree of the boot + * it was started on, not of the boot that resumed it. aura therefore hands Linux its device tree + * at a fixed address inside that range (/chosen/aura,hibernate-dt): the image does not save it, + * so after a resume the kernel finds this boot's tree at the address it was started with, and + * aura_hibernate_boot_fw_regions() reads a device's firmware regions from it. + * + * aura itself moves with the boot object, and with it its S2R mailbox: a restored kernel reads the + * mailbox of the boot it runs on from that tree too (aura_hibernate_boot_chosen_u64()). + */ + +#define pr_fmt(fmt) "aura-hibernate: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool aura_layout_fixed; +static u64 aura_layout[2]; +/* Where aura puts the device tree on every boot; 0 when it does not. */ +static phys_addr_t aura_boot_dt; +#define AURA_BOOT_DT_MAX SZ_2M + +/* + * Ranges kept out of the image, in page frames. Filled once at late_initcall, + * read through the arm64 hibernation platform hooks. A static table avoids + * register_nosave_region(), which allocates from memblock and is meant for + * setup_arch() time. + */ +#define AURA_NOSAVE_MAX 128 +static struct { unsigned long start, end; } aura_nosave[AURA_NOSAVE_MAX]; +static unsigned int aura_nosave_count; + +/* + * Apple silicon Macs run firmware on their coprocessors (storage, secure + * enclave, display, always-on processor, system controller, GPU and more). + * Restoring an image means a cold boot through the platform boot chain, which + * restarts that firmware, but most of the Linux drivers have no + * freeze/poweroff/restore path that rebuilds their mailbox, ring and DMA state + * from a cold start. The restored kernel would keep using stale state, so + * writing and restoring images stays disabled on these machines, except where + * the boot object gives the OS a fixed memory layout and the drivers rebuild + * their coprocessor state from the cold boot that restores the image (aura + * hib=1, see above). + */ +static bool aura_hibernate_blocked(void) +{ + return of_machine_is_compatible("apple,arm-platform") && !aura_layout_fixed; +} + +static bool aura_hibernate_pfn_nosave(unsigned long pfn) +{ + unsigned int i; + + for (i = 0; i < aura_nosave_count; i++) + if (pfn >= aura_nosave[i].start && pfn < aura_nosave[i].end) + return true; + return false; +} + +/* Whether the ranges kept out of the image cover all of [start, end). */ +static bool aura_hibernate_range_nosave(u64 start, u64 end) +{ + unsigned long pfn = PFN_DOWN(start), last = PFN_UP(end); + unsigned int i; + bool moved; + + while (pfn < last) { + moved = false; + for (i = 0; i < aura_nosave_count; i++) { + if (pfn >= aura_nosave[i].start && pfn < aura_nosave[i].end) { + pfn = aura_nosave[i].end; + moved = true; + } + } + if (!moved) + return false; + } + return true; +} + +/* + * The image restores every page to the frame it came from, so the boot that + * resumes it must give the kernel what the boot that wrote it did: on Apple + * machines at EL2 the same CTXR exec window (secondaries only execute inside + * it) and aura's fixed hibernation layout. They are added to the layout cookie + * of arm64 hibernation. + * + * aura's own image moves with the boot object, and its CPU release addresses + * and S2R mailbox with it, inside the range the layout keeps from Linux. The + * restored kernel uses neither address of the boot that wrote the image: its + * CPUs come back from the boot kernel's pen, a CPU started at its reset vector + * again finds the release target the boot kernel wrote (both kernels run at the + * window's fixed address), and the T6040 sleep driver takes this boot's mailbox + * from aura's tree after the restore. What has to match is the spin table's + * shape (the same CPUs, at the same offsets from the mailbox) and that this + * boot keeps all of it out of the image. + */ +static u64 aura_hibernate_layout(u64 h, struct arm64_hibernate_layout *lay) +{ + static const char * const chosen_props[] = { + "aura,hibernate-layout", "aura,hibernate-dt", + "aurora,ctxr-exec-window", + }; + u64 mbox = 0, layout[2], win[2]; + struct device_node *np; + const void *v; + u8 kept; + int i, len; + + np = of_find_node_by_path("/chosen"); + for (i = 0; np && i < ARRAY_SIZE(chosen_props); i++) { + v = of_get_property(np, chosen_props[i], &len); + if (!v) + continue; + h = arm64_hibernate_layout_mix(h, chosen_props[i], strlen(chosen_props[i])); + h = arm64_hibernate_layout_mix(h, v, len); + } + if (np && !of_property_read_u64(np, "aurora,s2r-mailbox", &mbox)) { + lay->have_base = true; + lay->base = mbox; + kept = aura_hibernate_pfn_nosave(PHYS_PFN(mbox)); + h = arm64_hibernate_layout_mix(h, "aurora,s2r-mailbox", + strlen("aurora,s2r-mailbox")); + h = arm64_hibernate_layout_mix(h, &kept, sizeof(kept)); + } + /* aura's layout less the OS window, only where all of it is kept out of the image */ + if (np && !of_property_read_u64_array(np, "aura,hibernate-layout", layout, 2) && + !of_property_read_u64_array(np, "aurora,ctxr-exec-window", win, 2) && + layout[0] <= win[0] && win[0] + win[1] <= layout[1]) { + lay->ex[lay->nex][0] = layout[0]; + lay->ex[lay->nex][1] = win[0]; + if (aura_hibernate_range_nosave(lay->ex[lay->nex][0], lay->ex[lay->nex][1])) + lay->nex++; + lay->ex[lay->nex][0] = win[0] + win[1]; + lay->ex[lay->nex][1] = layout[1]; + if (aura_hibernate_range_nosave(lay->ex[lay->nex][0], lay->ex[lay->nex][1])) + lay->nex++; + } + of_node_put(np); + return h; +} + +static const struct arm64_hibernate_platform aura_hibernate_platform = { + .pfn_nosave = aura_hibernate_pfn_nosave, + .layout = aura_hibernate_layout, +}; + +static void __init aura_nosave_add(unsigned long s, unsigned long e) +{ + if (s >= e) + return; + if (aura_nosave_count == AURA_NOSAVE_MAX) { + pr_warn("more than %d reserved ranges; hibernation stays disabled\n", AURA_NOSAVE_MAX); + aura_nosave_count++; + return; + } + if (aura_nosave_count < AURA_NOSAVE_MAX) { + aura_nosave[aura_nosave_count].start = s; + aura_nosave[aura_nosave_count].end = e; + aura_nosave_count++; + } +} + +static void __init aura_nosave_range(u64 start, u64 end, unsigned long kstart, unsigned long kend, + const char *what) +{ + unsigned long s = PFN_DOWN(start), e = PFN_UP(end); + + if (s >= e) + return; + /* The kernel image is Linux's own and is saved; split the range around it. */ + if (s < kend && e > kstart) { + if (s < kstart) + aura_nosave_add(s, kstart); + if (e > kend) + aura_nosave_add(kend, e); + } else { + aura_nosave_add(s, e); + } + pr_debug("nosave %s [%#llx, %#llx)\n", what, start, end); +} + +static bool aura_fw_region_kept(u64 start, u64 size) +{ + u64 end = start + size, a; + + if (end < start) + return false; + /* + * Inside the range aura keeps from Linux below its device tree slot, where the boot firmware places the + * firmware: reserved, and not saved in the image. Not the slot or aura's heap above it. + */ + if (start >= aura_layout[0] && end <= aura_boot_dt) + return true; + /* Otherwise Linux must neither own nor map any of it (no-map or not RAM). */ + for (a = start & PAGE_MASK; a < end; a += PAGE_SIZE) + if (memblock_is_map_memory(a)) + return false; + return true; +} + +/* + * Whether @fdt, mapped from @slot, is a tree aura wrote there: a valid header that fits the slot, and + * a /chosen naming this slot and the fixed layout @layout. + */ +static bool aura_slot_tree_valid(const void *fdt, phys_addr_t slot, const u64 layout[2]) +{ + const fdt64_t *v; + int node, len; + + if (fdt_check_header(fdt) || fdt_totalsize(fdt) > AURA_BOOT_DT_MAX) + return false; + node = fdt_path_offset(fdt, "/chosen"); + if (node < 0) + return false; + v = fdt_getprop(fdt, node, "aura,hibernate-dt", &len); + if (!v || len != sizeof(*v) || fdt64_ld(v) != slot) + return false; + v = fdt_getprop(fdt, node, "aura,hibernate-layout", &len); + return v && len == 2 * sizeof(*v) && fdt64_ld(&v[0]) == layout[0] && + fdt64_ld(&v[1]) == layout[1]; +} + +/* + * Reads the firmware memory-regions of @np (its memory-region phandles, each + * region's reg and the iommu-addresses entry naming @np) from the device tree + * of the running boot, which after a resume from disk is not the one the + * kernel was started with. Returns the number of regions, at most @max, or a + * negative errno: -ENODEV when aura does not hand the tree over at a fixed + * address. Only flat, untranslated layouts are accepted: @np's parent without + * dma-ranges, a /reserved-memory without a non-empty ranges. + */ +int aura_hibernate_boot_fw_regions(struct device_node *np, struct aura_fw_region *out, + unsigned int max) +{ + const fdt32_t *phandles, *p; + unsigned int n = 0; + int node, parent, len, ret, i; + u32 own; + char *path; + void *fdt; + + if (!aura_boot_dt || !np) + return -ENODEV; + fdt = memremap(aura_boot_dt, AURA_BOOT_DT_MAX, MEMREMAP_WB); + if (!fdt) + return -ENOMEM; + ret = -EINVAL; + /* The tree aura handed over on this boot, with the same fixed layout. */ + if (!aura_slot_tree_valid(fdt, aura_boot_dt, aura_layout)) + goto out; + + path = kasprintf(GFP_KERNEL, "%pOF", np); + if (!path) { + ret = -ENOMEM; + goto out; + } + node = fdt_path_offset(fdt, path); + kfree(path); + ret = -ENOENT; + if (node < 0) + goto out; + ret = -EINVAL; + own = fdt_get_phandle(fdt, node); + parent = fdt_parent_offset(fdt, node); + if (!own || parent < 0 || fdt_getprop(fdt, parent, "dma-ranges", NULL)) + goto out; + phandles = fdt_getprop(fdt, node, "memory-region", &len); + if (!phandles || len <= 0 || len % sizeof(*phandles)) + goto out; + + for (i = 0; i < len / (int)sizeof(*phandles); i++) { + int region, rparent, ac, sc, cells, pos, plen; + const fdt32_t *reg, *iommu; + u64 phys, size, iova = 0, length = 0; + bool found = false; + + region = fdt_node_offset_by_phandle(fdt, fdt32_to_cpu(phandles[i])); + if (region < 0) + goto out; + rparent = fdt_parent_offset(fdt, region); + if (rparent < 0) + goto out; + p = fdt_getprop(fdt, rparent, "ranges", &plen); + if (p && plen) + goto out; + ac = fdt_address_cells(fdt, rparent); + sc = fdt_size_cells(fdt, rparent); + reg = fdt_getprop(fdt, region, "reg", &plen); + if (ac < 1 || ac > 2 || sc < 1 || sc > 2 || !reg || + plen < (int)((ac + sc) * sizeof(*reg))) + goto out; + phys = fdt32_to_cpu(reg[0]); + if (ac == 2) + phys = phys << 32 | fdt32_to_cpu(reg[1]); + size = fdt32_to_cpu(reg[ac]); + if (sc == 2) + size = size << 32 | fdt32_to_cpu(reg[ac + 1]); + + /* Entries: owner phandle, then an address and a size in the owner's bus cells. */ + iommu = fdt_getprop(fdt, region, "iommu-addresses", &plen); + if (!iommu || plen <= 0 || plen % sizeof(*iommu)) + goto out; + cells = plen / sizeof(*iommu); + for (pos = 0; pos < cells;) { + u32 ph = fdt32_to_cpu(iommu[pos++]); + int owner = fdt_node_offset_by_phandle(fdt, ph), op, oac, osc; + + if (owner < 0) + goto out; + op = fdt_parent_offset(fdt, owner); + if (op < 0) + goto out; + oac = fdt_address_cells(fdt, op); + osc = fdt_size_cells(fdt, op); + if (oac < 1 || oac > 2 || osc < 1 || osc > 2 || oac + osc > cells - pos) + goto out; + if (ph == own) { + if (found) + goto out; + found = true; + iova = fdt32_to_cpu(iommu[pos]); + if (oac == 2) + iova = iova << 32 | fdt32_to_cpu(iommu[pos + 1]); + length = fdt32_to_cpu(iommu[pos + oac]); + if (osc == 2) + length = length << 32 | fdt32_to_cpu(iommu[pos + oac + 1]); + } + pos += oac + osc; + } + if (!found || length != size || !size) + goto out; + if (!aura_fw_region_kept(phys, size)) { + pr_warn("%pOF: this boot's firmware region [%#llx, +%#llx) is memory Linux owns\n", + np, phys, size); + ret = -EBUSY; + goto out; + } + if (n == max) { + ret = -E2BIG; + goto out; + } + out[n].phys = phys; + out[n].iova = iova; + out[n].size = size; + n++; + } + ret = n; +out: + memunmap(fdt); + return ret; +} +EXPORT_SYMBOL_GPL(aura_hibernate_boot_fw_regions); + +/* + * Reads the u64 property @name of /chosen from the device tree of the running boot, which after a + * resume from disk is not the one the kernel was started with. Returns 0, -ENODEV when aura does + * not hand the tree over at a fixed address, -EINVAL when the slot holds no tree for this layout, + * -ENOENT when the tree has no such property. + */ +int aura_hibernate_boot_chosen_u64(const char *name, u64 *val) +{ + const fdt64_t *v; + int node, len, ret; + void *fdt; + + if (!aura_boot_dt) + return -ENODEV; + fdt = memremap(aura_boot_dt, AURA_BOOT_DT_MAX, MEMREMAP_WB); + if (!fdt) + return -ENOMEM; + ret = -EINVAL; + if (!aura_slot_tree_valid(fdt, aura_boot_dt, aura_layout)) + goto out; + ret = -ENOENT; + node = fdt_path_offset(fdt, "/chosen"); + v = node < 0 ? NULL : fdt_getprop(fdt, node, name, &len); + if (!v || len != sizeof(*v)) + goto out; + *val = fdt64_ld(v); + ret = 0; +out: + memunmap(fdt); + return ret; +} +EXPORT_SYMBOL_GPL(aura_hibernate_boot_chosen_u64); + +struct aura_hibernate_reprobe { + struct work_struct work; + struct device *dev; +}; + +static void aura_hibernate_reprobe_work(struct work_struct *work) +{ + struct aura_hibernate_reprobe *rp = container_of(work, struct aura_hibernate_reprobe, work); + int ret; + + ret = device_reprobe(rp->dev); + /* A failed probe is not reported by device_reprobe() itself. */ + if (ret || !READ_ONCE(rp->dev->driver)) + dev_err(rp->dev, "probe after the resume from disk failed: %d\n", ret); + put_device(rp->dev); + kfree(rp); +} + +/* + * Unbinds @dev and probes it again once the tasks are thawed, for a driver whose co-processor + * the boot that resumed the image left halted: its unbind and its probe know how to drop a + * session that died under it and start the firmware anew, as at boot, and a restore callback + * cannot run either. The work runs on the freezable workqueue, which the restore finds frozen, + * so it starts after every device's restore and its complete step, when probing is allowed + * again. Built in, so that a driver module going away meanwhile leaves nothing behind. + */ +int aura_hibernate_reprobe_after_thaw(struct device *dev) +{ + struct aura_hibernate_reprobe *rp; + + rp = kzalloc_obj(*rp); + if (!rp) + return -ENOMEM; + INIT_WORK(&rp->work, aura_hibernate_reprobe_work); + rp->dev = get_device(dev); + queue_work(system_freezable_wq, &rp->work); + return 0; +} +EXPORT_SYMBOL_GPL(aura_hibernate_reprobe_after_thaw); + +static int __init aura_hibernate_init(void) +{ + unsigned long kstart = PFN_DOWN(__pa_symbol(_text)), kend = PFN_UP(__pa_symbol(_end)); + struct device_node *chosen, *rmem, *child; + unsigned int count = 0; + const void *fdt = initial_boot_params; + u64 layout[2], dt_slot; + int i, n; + + hibernation_set_platform_check(aura_hibernate_blocked, + "coprocessor firmware state is not restored from an image"); + arm64_hibernate_set_platform(&aura_hibernate_platform); + + /* Only a boot loader that keeps the layout fixed publishes it. */ + chosen = of_find_node_by_path("/chosen"); + if (!chosen || + of_property_read_u64_array(chosen, "aura,hibernate-layout", layout, 2)) { + of_node_put(chosen); + return 0; + } + /* + * The device tree slot. Trusted where this kernel's tree is, or, when a later stage (U-Boot and + * the EFI stub) handed the kernel its own copy of aura's tree, where the slot still holds aura's + * tree for this layout: those stages do not move the firmware segments, so aura's tree and the + * kernel's copy agree on them. + */ + if (!of_property_read_u64(chosen, "aura,hibernate-dt", &dt_slot)) { + bool in_layout = dt_slot >= layout[0] && dt_slot + AURA_BOOT_DT_MAX <= layout[1]; + void *slot_fdt; + + if (in_layout && dt_slot == __fdt_pointer) { + aura_boot_dt = dt_slot; + } else if (in_layout) { + slot_fdt = memremap(dt_slot, AURA_BOOT_DT_MAX, MEMREMAP_WB); + if (slot_fdt && aura_slot_tree_valid(slot_fdt, dt_slot, layout)) { + aura_boot_dt = dt_slot; + pr_info("device tree handed over at %pa; using aura's tree at %#llx\n", + &__fdt_pointer, dt_slot); + } else { + pr_warn("device tree at %pa, aura's slot %#llx holds no tree for this layout\n", + &__fdt_pointer, dt_slot); + } + if (slot_fdt) + memunmap(slot_fdt); + } else { + pr_warn("device tree slot %#llx outside the fixed layout\n", dt_slot); + } + } + of_node_put(chosen); + + if (fdt) { + n = fdt_num_mem_rsv(fdt); + for (i = 0; i < n; i++) { + u64 addr, size; + + if (fdt_get_mem_rsv(fdt, i, &addr, &size) || !size) + continue; + aura_nosave_range(addr, addr + size, kstart, kend, "memreserve"); + count++; + } + } + + rmem = of_find_node_by_path("/reserved-memory"); + for_each_available_child_of_node(rmem, child) { + struct resource r; + + if (of_property_read_bool(child, "reusable")) + continue; + if (of_address_to_resource(child, 0, &r)) + continue; + aura_nosave_range(r.start, r.end + 1, kstart, kend, "reserved-memory"); + count++; + } + of_node_put(rmem); + + if (aura_nosave_count > AURA_NOSAVE_MAX) { + aura_nosave_count = 0; + return 0; + } + aura_layout[0] = layout[0]; + aura_layout[1] = layout[1]; + aura_layout_fixed = true; + pr_info("fixed boot layout [%#llx, %#llx), %u reserved ranges kept out of the image%s\n", + layout[0], layout[1], count, + aura_boot_dt ? ", device tree at a fixed address" : ""); + return 0; +} +late_initcall(aura_hibernate_init); diff --git a/drivers/soc/apple/dockchannel.c b/drivers/soc/apple/dockchannel.c index 77dc01f8b4359..291c5cc3ce319 100644 --- a/drivers/soc/apple/dockchannel.c +++ b/drivers/soc/apple/dockchannel.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,13 @@ #define DATA_RX32 0x28 #define DATA_RX_COUNT 0x2c +/* Bits of dockchannel::armed: the interrupt is enabled and has not fired since. */ +#define DOCKCHANNEL_TX_ARMED 0 +#define DOCKCHANNEL_RX_ARMED 1 + +/* Receive interrupts below the threshold in a row before they are handled anyway. */ +#define DOCKCHANNEL_RX_SPURIOUS_MAX 16 + struct dockchannel { struct device *dev; int tx_irq; @@ -56,6 +64,20 @@ struct dockchannel { struct completion tx_comp; struct completion rx_comp; + /* + * Each interrupt is enabled by whoever arms it and disabled exactly once + * afterwards, by its hard handler or by whoever gives up waiting, so the + * enable depth stays balanced. + */ + unsigned long armed; + /* The receive threshold last programmed. */ + u32 rx_thresh; + unsigned int rx_spurious; + /* Receive was stopped by dockchannel_await(..., 0) from outside the callback. */ + bool rx_stopped; + /* The receive interrupt thread, which runs the client callback. */ + struct task_struct *rx_thread; + void *cookie; void (*data_available)(void *cookie, size_t avail); }; @@ -70,23 +92,77 @@ struct dockchannel_common { /* Dockchannel FIFO functions */ +static void dockchannel_arm(struct dockchannel *dockchannel, int bit, int irq) +{ + if (!test_and_set_bit(bit, &dockchannel->armed)) + enable_irq(irq); +} + +/* + * Never disable_irq() here: a client's receive callback runs in the receive + * interrupt thread and waits for data from it, and disable_irq() would wait + * for that thread to finish, i.e. for itself. + */ +static void dockchannel_disarm(struct dockchannel *dockchannel, int bit, int irq) +{ + if (test_and_clear_bit(bit, &dockchannel->armed)) + disable_irq_nosync(irq); +} + static irqreturn_t dockchannel_tx_irq(int irq, void *data) { struct dockchannel *dockchannel = data; + /* Not armed: a sender giving up disables it. */ + if (!test_and_clear_bit(DOCKCHANNEL_TX_ARMED, &dockchannel->armed)) + return IRQ_HANDLED; disable_irq_nosync(irq); complete(&dockchannel->tx_comp); return IRQ_HANDLED; } +/* + * An interrupt with less data than the threshold that was programmed means + * that the FIFO does not hold that threshold: after a resume from disk the + * configuration registers hold what the boot loader or the boot kernel left, + * while the restored interrupt state says that receive is armed. Program it + * again and keep waiting, instead of waking a client to read data that is + * not there. Only a kernel restored from an image is in that position: any + * other takes every receive interrupt as it comes. + */ +static bool dockchannel_rx_spurious(struct dockchannel *dockchannel) +{ + u32 thresh = READ_ONCE(dockchannel->rx_thresh); + + if (!pm_hibernate_is_restored()) + return false; + if (readl_relaxed(dockchannel->data_base + DATA_RX_COUNT) >= thresh) + goto real; + writel_relaxed(thresh, dockchannel->config_base + CONFIG_RX_THRESH); + if (readl_relaxed(dockchannel->data_base + DATA_RX_COUNT) >= thresh) + goto real; + if (++dockchannel->rx_spurious <= DOCKCHANNEL_RX_SPURIOUS_MAX) + return true; + dev_warn_once(dockchannel->dev, + "receive interrupt keeps firing below its threshold (%u)\n", thresh); +real: + dockchannel->rx_spurious = 0; + return false; +} + static irqreturn_t dockchannel_rx_irq(int irq, void *data) { struct dockchannel *dockchannel = data; + if (dockchannel_rx_spurious(dockchannel)) + return IRQ_HANDLED; + /* Not armed: whoever is disarming it disables it. */ + if (!test_and_clear_bit(DOCKCHANNEL_RX_ARMED, &dockchannel->armed)) + return IRQ_HANDLED; disable_irq_nosync(irq); - if (dockchannel->awaiting) { + if (READ_ONCE(dockchannel->awaiting)) { return IRQ_WAKE_THREAD; } else { complete(&dockchannel->rx_comp); @@ -97,9 +173,15 @@ static irqreturn_t dockchannel_rx_irq(int irq, void *data) static irqreturn_t dockchannel_rx_irq_thread(int irq, void *data) { struct dockchannel *dockchannel = data; - size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + size_t avail; + + WRITE_ONCE(dockchannel->rx_thread, current); + /* Woken just before a stop: the client asked for nothing more. */ + if (READ_ONCE(dockchannel->rx_stopped) || !dockchannel->data_available) + return IRQ_HANDLED; - dockchannel->awaiting = false; + avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + WRITE_ONCE(dockchannel->awaiting, false); dockchannel->data_available(dockchannel->cookie, avail); return IRQ_HANDLED; @@ -119,11 +201,12 @@ int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t co writel_relaxed(threshold, dockchannel->config_base + CONFIG_TX_THRESH); reinit_completion(&dockchannel->tx_comp); - enable_irq(dockchannel->tx_irq); + dockchannel_arm(dockchannel, DOCKCHANNEL_TX_ARMED, dockchannel->tx_irq); if (!wait_for_completion_timeout(&dockchannel->tx_comp, msecs_to_jiffies(DOCKCHANNEL_TX_TIMEOUT_MS))) { - disable_irq(dockchannel->tx_irq); + dockchannel_disarm(dockchannel, DOCKCHANNEL_TX_ARMED, + dockchannel->tx_irq); return -ETIMEDOUT; } @@ -147,6 +230,11 @@ int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t co } EXPORT_SYMBOL(dockchannel_send); +/* + * Called by the client's receive callback, in the receive interrupt thread: + * waiting for the rest of a frame here re-arms the receive interrupt, whose + * hard handler then completes the wait instead of waking the thread. + */ int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count) { size_t left = count; @@ -159,13 +247,15 @@ int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count) if (avail == 0) { size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + WRITE_ONCE(dockchannel->rx_thresh, threshold); writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); reinit_completion(&dockchannel->rx_comp); - enable_irq(dockchannel->rx_irq); + dockchannel_arm(dockchannel, DOCKCHANNEL_RX_ARMED, dockchannel->rx_irq); if (!wait_for_completion_timeout(&dockchannel->rx_comp, msecs_to_jiffies(DOCKCHANNEL_RX_TIMEOUT_MS))) { - disable_irq(dockchannel->rx_irq); + dockchannel_disarm(dockchannel, DOCKCHANNEL_RX_ARMED, + dockchannel->rx_irq); return -ETIMEDOUT; } @@ -189,6 +279,24 @@ int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count) } EXPORT_SYMBOL(dockchannel_recv); +/* + * Stop receiving. From outside the callback this also waits for a callback + * that is running to return, and undoes the re-arm it may have done meanwhile; + * the thread does not call the client again until receive is restarted. + */ +static void dockchannel_rx_stop(struct dockchannel *dockchannel) +{ + WRITE_ONCE(dockchannel->rx_stopped, true); + WRITE_ONCE(dockchannel->awaiting, false); + dockchannel_disarm(dockchannel, DOCKCHANNEL_RX_ARMED, dockchannel->rx_irq); + if (current == READ_ONCE(dockchannel->rx_thread)) + return; + + synchronize_irq(dockchannel->rx_irq); + WRITE_ONCE(dockchannel->awaiting, false); + dockchannel_disarm(dockchannel, DOCKCHANNEL_RX_ARMED, dockchannel->rx_irq); +} + int dockchannel_await(struct dockchannel *dockchannel, void (*callback)(void *cookie, size_t avail), void *cookie, size_t count) @@ -196,16 +304,24 @@ int dockchannel_await(struct dockchannel *dockchannel, size_t threshold = min((size_t)dockchannel->fifo_size, count); if (!count) { - dockchannel->awaiting = false; - disable_irq(dockchannel->rx_irq); + dockchannel_rx_stop(dockchannel); return 0; } + if (current == READ_ONCE(dockchannel->rx_thread)) { + /* The callback re-arms itself; a stop that came meanwhile wins. */ + if (READ_ONCE(dockchannel->rx_stopped)) + return 0; + } else { + WRITE_ONCE(dockchannel->rx_stopped, false); + } + dockchannel->data_available = callback; dockchannel->cookie = cookie; - dockchannel->awaiting = true; + WRITE_ONCE(dockchannel->awaiting, true); + WRITE_ONCE(dockchannel->rx_thresh, threshold); writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); - enable_irq(dockchannel->rx_irq); + dockchannel_arm(dockchannel, DOCKCHANNEL_RX_ARMED, dockchannel->rx_irq); return threshold; } @@ -415,6 +531,33 @@ static void dockchannel_remove(struct platform_device *pdev) writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); } +/* + * After a resume from disk the interrupt mask register holds what the boot + * kernel left in it (nothing, if it never loaded this driver), while the + * restored interrupt descriptors still describe the image's state. Write the + * image's state back before the channel clients restore; an unmasked receive + * interrupt that the hardware has masked would never fire. + */ +static int dockchannel_restore_noirq(struct device *dev) +{ + struct dockchannel_common *dcc = dev_get_drvdata(dev); + u32 mask = 0; + int hwirq; + + for (hwirq = 0; hwirq < DOCKCHANNEL_MAX_IRQ; hwirq++) { + unsigned int virq = irq_find_mapping(dcc->domain, hwirq); + + if (virq && !irqd_irq_masked(irq_get_irq_data(virq))) + mask |= BIT(hwirq); + } + writel_relaxed(mask, dcc->irq_base + IRQ_MASK); + return 0; +} + +static const struct dev_pm_ops dockchannel_pm_ops = { + .restore_noirq = pm_sleep_ptr(dockchannel_restore_noirq), +}; + static const struct of_device_id dockchannel_of_match[] = { { .compatible = "apple,dockchannel" }, {}, @@ -425,6 +568,7 @@ static struct platform_driver dockchannel_driver = { .driver = { .name = "dockchannel", .of_match_table = dockchannel_of_match, + .pm = pm_sleep_ptr(&dockchannel_pm_ops), }, .probe = dockchannel_probe, .remove = dockchannel_remove, diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index e18e84bb1986a..86979c30d49b1 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -6,6 +6,16 @@ //! capture in flight did not wake. Before the system suspends, end any capture //! in progress and hold new ones until it has resumed. The hwrng fill thread //! is not frozen either, so entropy requests are held over the same window. +//! +//! A hibernation also holds every request Linux starts, and the enclave's +//! xART writes are refused, from its prepare until the kernel knows how it +//! ended. The image must not miss host or gigalocker state written after the +//! snapshot, and after a resume from disk on a cold boot the enclave is a +//! freshly booted sepOS that must not see a message of the old session before +//! the new registration. The PM core tells a resume from the image from an +//! aborted hibernation or the wake of a hybrid sleep, and the PMP, which the +//! boot loader halts on a cold boot, whether that resume went through one +//! (`pm_shim.c`); the device is then probed again. //! The notifier itself is in `pm_shim.c`; the capture and entropy handling is //! on [`SepData`]. //! @@ -29,7 +39,7 @@ static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { fn sep_pm_register( dev: *mut c_void, - event: unsafe extern "C" fn(bool), + event: unsafe extern "C" fn(c_int) -> c_int, sleep: unsafe extern "C" fn() -> c_int, wake: unsafe extern "C" fn(), told: unsafe extern "C" fn() -> bool, @@ -50,6 +60,14 @@ extern "C" { pub(crate) static sep_pm_ops: kernel::bindings::dev_pm_ops; } +// Sleep events from `pm_shim.c`; must match `SEP_PM_*` in `shim.h`. +const EVENT_SLEEP: c_int = 0; +const EVENT_WAKE: c_int = 1; +const EVENT_HIBERNATE: c_int = 2; +const EVENT_HIBERNATE_WARM: c_int = 3; +const EVENT_HIBERNATE_COLD: c_int = 4; +const EVENT_RESTORE: c_int = 5; + /// Who may use the mailbox around a firmware-driven suspend. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum SleepGate { @@ -164,6 +182,14 @@ trait SleepHolds { fn hold_captures(&self); fn release_captures(&self); fn release_entropy(&self); + /// Holds every request Linux starts and refuses the enclave's xART + /// writes, or lifts that, around a hibernation. + fn hold_session(&self, held: bool); + /// The enclave was booted anew under a hibernation image: the session + /// is gone. Returns whether the device should be probed again. + fn session_lost(&self) -> bool; + /// Whether this kernel registered with the enclave. + fn attached(&self) -> bool; } impl SleepHolds for SepData { @@ -179,32 +205,70 @@ impl SleepHolds for SepData { fn release_entropy(&self) { self.rng_sleep_finished(); } + fn hold_session(&self, held: bool) { + self.hibernation_hold(held); + } + fn session_lost(&self) -> bool { + self.lose_session() + } + fn attached(&self) -> bool { + SepData::attached(self) + } } -/// `entering`: the system is about to sleep; otherwise it has resumed, or -/// the suspend was aborted. -fn sleep_event(sep: &impl SleepHolds, entering: bool) { - if entering { - sep.hold_entropy(); - sep.hold_captures(); - } else { - sep.release_captures(); - sep.release_entropy(); +/// Takes or releases the holds for one sleep event. Returns 1 when the +/// device must be probed again, or for `EVENT_RESTORE` when the restore must +/// be refused; 0 otherwise. +fn sleep_event(sep: &impl SleepHolds, event: c_int) -> c_int { + match event { + EVENT_SLEEP => { + sep.hold_entropy(); + sep.hold_captures(); + } + EVENT_HIBERNATE => { + sep.hold_entropy(); + sep.hold_captures(); + sep.hold_session(true); + } + EVENT_WAKE => { + sep.release_captures(); + sep.release_entropy(); + } + // This kernel is about to load a hibernation image: refused (1) if + // it registered with the enclave, see `pm_shim.c`. + EVENT_RESTORE => { + if sep.attached() { + return 1; + } + sep.hold_entropy(); + sep.hold_captures(); + sep.hold_session(true); + } + EVENT_HIBERNATE_WARM => { + sep.hold_session(false); + sep.release_captures(); + sep.release_entropy(); + } + // The holds stay: the removal ahead of the new probe releases them + // into a session that refuses everything. + EVENT_HIBERNATE_COLD => return sep.session_lost() as c_int, + _ => {} } + 0 } /// # Safety /// Called by `pm_shim.c` from the PM notifier chain, in process context. -unsafe extern "C" fn pm_event(entering: bool) { +unsafe extern "C" fn pm_event(event: c_int) -> c_int { let ptr = SEP.load(Ordering::Acquire); if ptr.is_null() { - return; + return 0; } // SAFETY: produced by `into_foreign` in `register`, and reclaimed by // `unregister` only after `sep_pm_unregister` has waited out any running // notifier call, so it is live for this borrow. let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; - sleep_event(&*sep, entering); + sleep_event(&*sep, event) } /// Runs `f` on the registered instance. Only for callbacks that @@ -270,6 +334,7 @@ pub(crate) fn register(sep: Arc) -> Result<()> { // SAFETY: the shim stores the callback and registers a notifier block // that lives for the module's lifetime. + // `dev` is the bound SEP device: the shim drops it in `sep_pm_unregister`. let rc = unsafe { sep_pm_register(dev, pm_event, pm_sleep, pm_wake, pm_told) }; if rc != 0 { let raw = SEP.swap(core::ptr::null_mut(), Ordering::AcqRel); @@ -308,10 +373,15 @@ mod tests { const HOLD_CAPTURES: u8 = 2; const RELEASE_CAPTURES: u8 = 3; const RELEASE_ENTROPY: u8 = 4; + const HOLD_SESSION: u8 = 5; + const RELEASE_SESSION: u8 = 6; + const LOSE_SESSION: u8 = 7; struct Calls { made: RefCell<[u8; 8]>, count: Cell, + lose: Cell, + attached: Cell, } impl Calls { @@ -319,6 +389,8 @@ mod tests { Calls { made: RefCell::new([0; 8]), count: Cell::new(0), + lose: Cell::new(true), + attached: Cell::new(false), } } fn call(&self, which: u8) { @@ -344,6 +416,16 @@ mod tests { fn release_entropy(&self) { self.call(RELEASE_ENTROPY); } + fn hold_session(&self, held: bool) { + self.call(if held { HOLD_SESSION } else { RELEASE_SESSION }); + } + fn session_lost(&self) -> bool { + self.call(LOSE_SESSION); + self.lose.get() + } + fn attached(&self) -> bool { + self.attached.get() + } } #[test] @@ -376,7 +458,7 @@ mod tests { #[test] fn going_to_sleep_holds_entropy_then_captures() { let calls = Calls::new(); - sleep_event(&calls, true); + assert_eq!(sleep_event(&calls, EVENT_SLEEP), 0); assert_eq!( calls.made(), [HOLD_ENTROPY, HOLD_CAPTURES, 0, 0, 0, 0, 0, 0] @@ -386,9 +468,9 @@ mod tests { #[test] fn sleep_holds_stay_until_the_post_event() { let calls = Calls::new(); - sleep_event(&calls, true); + assert_eq!(sleep_event(&calls, EVENT_SLEEP), 0); assert_eq!(calls.count.get(), 2); - sleep_event(&calls, false); + assert_eq!(sleep_event(&calls, EVENT_WAKE), 0); assert_eq!( calls.made(), [HOLD_ENTROPY, HOLD_CAPTURES, RELEASE_CAPTURES, RELEASE_ENTROPY, 0, 0, 0, 0] @@ -398,10 +480,83 @@ mod tests { #[test] fn waking_releases_captures_then_entropy() { let calls = Calls::new(); - sleep_event(&calls, false); + assert_eq!(sleep_event(&calls, EVENT_WAKE), 0); assert_eq!( calls.made(), [RELEASE_CAPTURES, RELEASE_ENTROPY, 0, 0, 0, 0, 0, 0] ); } + + #[test] + fn a_hibernation_holds_the_session_last() { + let calls = Calls::new(); + assert_eq!(sleep_event(&calls, EVENT_HIBERNATE), 0); + assert_eq!( + calls.made(), + [HOLD_ENTROPY, HOLD_CAPTURES, HOLD_SESSION, 0, 0, 0, 0, 0] + ); + } + + #[test] + fn a_warm_end_of_a_hibernation_lifts_the_session_hold_first() { + let calls = Calls::new(); + assert_eq!(sleep_event(&calls, EVENT_HIBERNATE), 0); + assert_eq!(sleep_event(&calls, EVENT_HIBERNATE_WARM), 0); + assert_eq!( + calls.made(), + [ + HOLD_ENTROPY, + HOLD_CAPTURES, + HOLD_SESSION, + RELEASE_SESSION, + RELEASE_CAPTURES, + RELEASE_ENTROPY, + 0, + 0 + ] + ); + } + + #[test] + fn a_cold_resume_keeps_the_holds_and_asks_for_a_probe() { + let calls = Calls::new(); + assert_eq!(sleep_event(&calls, EVENT_HIBERNATE), 0); + assert_eq!(sleep_event(&calls, EVENT_HIBERNATE_COLD), 1); + assert_eq!( + calls.made(), + [HOLD_ENTROPY, HOLD_CAPTURES, HOLD_SESSION, LOSE_SESSION, 0, 0, 0, 0] + ); + } + + #[test] + fn a_second_cold_resume_asks_for_no_second_probe() { + let calls = Calls::new(); + calls.lose.set(false); + assert_eq!(sleep_event(&calls, EVENT_HIBERNATE_COLD), 0); + } + + #[test] + fn a_restore_over_an_attached_enclave_is_refused_without_holds() { + let calls = Calls::new(); + calls.attached.set(true); + assert_eq!(sleep_event(&calls, EVENT_RESTORE), 1); + assert_eq!(calls.count.get(), 0); + } + + #[test] + fn a_restore_without_a_session_holds_like_a_hibernation() { + let calls = Calls::new(); + assert_eq!(sleep_event(&calls, EVENT_RESTORE), 0); + assert_eq!( + calls.made(), + [HOLD_ENTROPY, HOLD_CAPTURES, HOLD_SESSION, 0, 0, 0, 0, 0] + ); + } + + #[test] + fn an_unknown_event_changes_nothing() { + let calls = Calls::new(); + assert_eq!(sleep_event(&calls, 99), 0); + assert_eq!(calls.count.get(), 0); + } } diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index c1d415c0bd744..23c371ec7a9d4 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -1,12 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * System sleep for the SEP driver: the notifier only forwards "about to sleep" - * and "back from sleep" to Rust, which ends any Touch ID capture before the - * system suspends (see pm.rs), and the enclave's power domains are kept on - * across sleep. The device's suspend and resume callbacks are the firmware - * suspend's: the enclave is told, and gates its own domain, whose state word is - * read here. On T8140 at EL2 the sleep driver sends the request at the - * irreversible commit boundary. Only errors before send may refuse. + * System sleep for the SEP driver: the notifier forwards the sleep events to + * Rust, which ends any Touch ID capture before the system suspends (see + * pm.rs); after a resume from disk on a cold boot it probes the device again; + * and the enclave's power domains are kept on across sleep. The device's + * suspend and resume callbacks are the firmware suspend's: the enclave is + * told, and gates its own domain, whose state word is read here. On T8140 at + * EL2 the sleep driver sends the request at the irreversible commit boundary. + * Only errors before send may refuse. */ #include #include @@ -23,9 +24,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -42,33 +45,128 @@ module_param(sleep_refuse_before_send, bool, 0644); MODULE_PARM_DESC(sleep_refuse_before_send, "Test only: refuse the next prepared deep sleep before sending SEP SLEEP"); -static void (*sep_pm_event)(bool entering); +static int (*sep_pm_event)(int event); +static struct device *sep_pm_dev; static int (*sep_pm_sleep)(void); static void (*sep_pm_wake)(void); static bool (*sep_pm_told)(void); /* In the device's .suspend, the only caller that can leave the request to the sleep. */ static bool sep_pm_suspending; +/* The marker dt.rs leaves on the SEP node once the registration is sent. */ +#define SEP_REGISTERED_PROP "apple,sep-shmem-registered-iova" + +/* + * A resume from disk on a cold boot: the enclave runs a freshly booted sepOS + * that has never seen this kernel's registration, so the restored session is + * dead. The device is probed again once the tasks are thawed, as at boot: + * probe sends a new registration (the marker is gone with the session it + * guarded), opens the xART store and loads the host state from disk again (the + * boot that resumed this image may have changed both), and recreates + * /dev/sep-bio. + */ +static void sep_pm_schedule_reprobe(struct device *dev) +{ + struct device_node *np = dev->of_node; + struct property *prop; + int ret; + + /* + * Only here, with the cold boot established: removing the marker on a + * live enclave would let a second registration park it. + */ + prop = np ? of_find_property(np, SEP_REGISTERED_PROP, NULL) : NULL; + if (prop && of_remove_property(np, prop)) { + dev_err(dev, "resume from disk: the registration marker could not be removed; Touch ID stays down until a reboot\n"); + return; + } + + if (!IS_ENABLED(CONFIG_APPLE_AURA_HIBERNATE)) { + dev_err(dev, "resume from disk: no aura hibernation support to probe the SEP again; Touch ID stays down until a reboot\n"); + return; + } + + dev_info(dev, "resume from disk: attaching the restarted enclave again\n"); + ret = aura_hibernate_reprobe_after_thaw(dev); + if (ret) + dev_err(dev, "resume from disk: probing the SEP again could not be queued (%d); Touch ID stays down until a reboot\n", + ret); +} + static int sep_pm_notify(struct notifier_block *nb, unsigned long action, void *data) { - void (*event)(bool entering) = READ_ONCE(sep_pm_event); + int (*event)(int event) = READ_ONCE(sep_pm_event); + struct device *dev = READ_ONCE(sep_pm_dev); + int cold; if (!event) return NOTIFY_DONE; switch (action) { case PM_SUSPEND_PREPARE: + sep_diag_sleep(true); + event(SEP_PM_SLEEP); + return NOTIFY_OK; + case PM_POST_SUSPEND: + event(SEP_PM_WAKE); + sep_diag_sleep(false); + return NOTIFY_OK; case PM_HIBERNATION_PREPARE: + sep_diag_sleep(true); + event(SEP_PM_HIBERNATE); + return NOTIFY_OK; case PM_RESTORE_PREPARE: + /* + * This kernel is about to load an image. If it registered with the + * enclave, the enclave keeps that session and may DMA into memory + * the image overwrites, and the restored kernel would register a + * second time: refuse the restore (this boot goes on instead). + */ + if (event(SEP_PM_RESTORE) > 0) { + pr_err("apple_sep: this kernel attached the SEP; refusing to restore a hibernation image over it\n"); + return NOTIFY_BAD; + } + /* Not before the refusal: no PM_POST_RESTORE comes to the one who refused. */ sep_diag_sleep(true); - event(true); return NOTIFY_OK; - case PM_POST_SUSPEND: - case PM_POST_HIBERNATION: case PM_POST_RESTORE: - event(false); + event(SEP_PM_HIBERNATE_WARM); + sep_diag_sleep(false); + return NOTIFY_OK; + case PM_POST_HIBERNATION: sep_diag_sleep(false); + /* + * Ends an aborted hibernation, the wake of a hybrid sleep and a + * resume from the image alike. iBoot boots sepOS anew on every + * boot and the SEP has no run bit the AP can read, so only a + * resume from the image that another boot loaded can have gone + * through a boot; the PM core knows whether it did (a test + * resume does not). A hybrid sleep's wake is never taken for a + * cold boot. + */ + if (!pm_hibernate_is_restored()) { + event(SEP_PM_HIBERNATE_WARM); + return NOTIFY_OK; + } + cold = pm_hibernate_restored_cold(); + if (cold < 0) { + /* + * Cold or warm cannot be told. Sending to a freshly booted + * enclave, or registering again with a live one, are both + * worse than losing Touch ID until a reboot. + */ + if (dev) + dev_err(dev, "hibernation: no record of the boot that loaded the image, cannot tell a cold boot; the SEP session is dropped until a reboot\n"); + event(SEP_PM_HIBERNATE_COLD); + return NOTIFY_OK; + } + if (cold) { + if (event(SEP_PM_HIBERNATE_COLD) > 0 && dev) + sep_pm_schedule_reprobe(dev); + return NOTIFY_OK; + } + event(SEP_PM_HIBERNATE_WARM); return NOTIFY_OK; default: return NOTIFY_DONE; @@ -474,11 +572,12 @@ const struct dev_pm_ops sep_pm_ops = { .suspend_noirq = pm_sleep_ptr(sep_pm_suspend_noirq), }; -int sep_pm_register(void *ptr, void (*event)(bool entering), int (*sleep)(void), +int sep_pm_register(void *ptr, int (*event)(int event), int (*sleep)(void), void (*wake)(void), bool (*told)(void)) { int ret; + WRITE_ONCE(sep_pm_dev, (struct device *)ptr); WRITE_ONCE(sep_pm_event, event); WRITE_ONCE(sep_pm_sleep, sleep); WRITE_ONCE(sep_pm_wake, wake); @@ -489,6 +588,7 @@ int sep_pm_register(void *ptr, void (*event)(bool entering), int (*sleep)(void), WRITE_ONCE(sep_pm_sleep, NULL); WRITE_ONCE(sep_pm_wake, NULL); WRITE_ONCE(sep_pm_told, NULL); + WRITE_ONCE(sep_pm_dev, NULL); return ret; } @@ -503,6 +603,7 @@ void sep_pm_unregister(void) WRITE_ONCE(sep_pm_sleep, NULL); WRITE_ONCE(sep_pm_wake, NULL); WRITE_ONCE(sep_pm_told, NULL); + WRITE_ONCE(sep_pm_dev, NULL); } static void sep_pm_release_domains(void *data) diff --git a/drivers/soc/apple/pmp-export.c b/drivers/soc/apple/pmp-export.c index 25629d0581706..b34dfe599b6d1 100644 --- a/drivers/soc/apple/pmp-export.c +++ b/drivers/soc/apple/pmp-export.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -20,27 +21,68 @@ static struct device *pmp_device; /* The firmware is stopped for a system sleep. */ static bool pmp_asleep; -int apple_pmp_register(struct device *dev, const void *data) +/* + * After a hibernation restore the boot loader has cold-restarted the PMP + * (image resident in PMS_SRAM, ASC CPU halted) and nothing replays the + * probe-time bring-up, so the firmware stays down (the display gate and + * cpufreq both wait on a PMP that never reports running). The module cannot be + * reloaded (the power-vote bridge is retained until reboot), so rebuild the + * firmware session in place. + * + * The restored kernel resumes inside hibernate() with in_suspend == 0 and + * sends PM_POST_HIBERNATION, not PM_POST_RESTORE (that one only reaches the + * boot kernel when it fails to restore). PM_POST_HIBERNATION also ends an + * aborted hibernation, where the firmware never stopped: restore() reads the + * ASC CPU run bit and leaves a running firmware alone. + */ +static int apple_pmp_pm_notify(struct notifier_block *nb, unsigned long action, + void *unused) { - int ret = 0; - + if (action != PM_POST_HIBERNATION && action != PM_POST_RESTORE) + return NOTIFY_DONE; + /* Serialize against unregister in Drop, as the power-command path does. */ guard(mutex)(&pmp_lock); if (pmp_data) - ret = -EBUSY; - else { - pmp_data = data; + apple_pmp_restore(pmp_data); + return NOTIFY_DONE; +} + +static struct notifier_block apple_pmp_pm_nb = { + .notifier_call = apple_pmp_pm_notify, + /* Ahead of every consumer's restore at priority 0 or below (ANE, DCP, GPU). */ + .priority = 10, +}; + +int apple_pmp_register(struct device *dev, const void *data) +{ + scoped_guard(mutex, &pmp_lock) { + if (pmp_data) + return -EBUSY; + WRITE_ONCE(pmp_data, data); pmp_device = dev; } - return ret; + /* + * Outside pmp_lock: the notifier chain's lock is taken around the + * callback, which takes pmp_lock, so nesting them here would invert the + * order. The callback re-checks pmp_data under the lock. + */ + register_pm_notifier(&apple_pmp_pm_nb); + return 0; } void apple_pmp_unregister(const void *data) { - guard(mutex)(&pmp_lock); - if (pmp_data == data) { - pmp_data = NULL; - pmp_device = NULL; + bool was_registered = false; + + scoped_guard(mutex, &pmp_lock) { + if (pmp_data == data) { + WRITE_ONCE(pmp_data, NULL); + pmp_device = NULL; + was_registered = true; + } } + if (was_registered) + unregister_pm_notifier(&apple_pmp_pm_nb); } int apple_pmp_link_device(struct device *consumer) @@ -152,11 +194,13 @@ static int apple_pmp_commit_stop_fn(void *unused) /* * Retained J713 power votes have no teardown. On T8140 an unbind leaves the firmware running, * and a commit with it running reset the SoC: its sleep registration is what refuses - * then, every sleep until reboot, since a rebind can't register again. + * then, every sleep until reboot, since a rebind can't register again. The T6040 sleep + * controller keeps the same registration, never undone. */ bool apple_pmp_keep_bound(void) { - return apple_pmp_display_retained_board() || of_machine_is_compatible("apple,t8140"); + return apple_pmp_display_retained_board() || of_machine_is_compatible("apple,t8140") || + of_machine_is_compatible("apple,t6040"); } /* Whether the sleep's last step takes the PMP: registered with the sleep driver, once. */ @@ -211,6 +255,11 @@ static int apple_pmp_suspend_late(struct device *dev) dev_err(dev, "no stop at the sleep's commit: refusing the sleep\n"); return -EBUSY; } + /* An idled PMP comes back only through a real sleep, which a dry run never reaches. */ + if (!at_commit && !apple_sleep_power_loss()) { + dev_warn(dev, "dry run: not idled, the sleep controller can't take it at the commit\n"); + return 0; + } apple_pmp_report_sleep(dev, at_commit); ret = at_commit ? apple_pmp_commit_prepare(dev) : apple_pmp_sleep(pmp_data); if (ret) { diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 337be4192474e..6de39b8b76c55 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -181,10 +181,14 @@ struct PmpData { display_error: AtomicI32, /// See [`PmpHwConfig::sram_backup`]. sram_backup: Option<(usize, usize)>, + /// The SoC configuration, kept so the hibernation restore can re-run the + /// probe-time bring-up (the boot loader cold-restarts the PMP on an S4 + /// resume and nothing else replays it). + cfg: &'static PmpHwConfig, } impl PmpData { - fn new(dev: &platform::Device>, cfg: &PmpHwConfig) -> Result> { + fn new(dev: &platform::Device>, cfg: &'static PmpHwConfig) -> Result> { let pmp_req = dev.io_request_by_name(c"pmp").ok_or(EINVAL)?; let pmp_mmio = pmp_req.iomap_sized::()?.into_devres()?; let asc_req = dev.io_request_by_name(c"asc").ok_or(EINVAL)?; @@ -202,11 +206,93 @@ impl PmpData { display: AtomicPtr::new(core::ptr::null_mut()), display_error: AtomicI32::new(0), sram_backup: cfg.sram_backup, + cfg, } ), GFP_KERNEL, ) } + + /// Build the SRAM boot-argument patch list from the device-tree node, as + /// `probe()` does. Re-derivable on a hibernation restore (the DT is static). + fn build_patches(&self) -> Result> { + let node = self.dev.fwnode().ok_or(EIO)?; + let dvid = node + .property_read(c"apple,dram-vendor-id") + .required_by(&self.dev)?; + let bdid = node.property_read(c"apple,board-id").required_by(&self.dev)?; + let mut patches = KVec::with_capacity(9, GFP_KERNEL)?; + patches.push((from_fourcc(b"BDID"), bdid), GFP_KERNEL)?; + patches.push((from_fourcc(b"DVID"), dvid), GFP_KERNEL)?; + for (tag, prop) in OPTIONAL_BOOTARGS.iter() { + if let Some(v) = node.property_read::(prop).optional() { + patches.push((from_fourcc(tag), v), GFP_KERNEL)?; + } + } + if let Some(v) = node.property_read::(c"apple,pmc-pmgr").optional() { + patches.push((from_fourcc(b"PMCV"), v & 1), GFP_KERNEL)?; + patches.push((from_fourcc(b"PMCB"), (v >> 3) & 1), GFP_KERNEL)?; + } + Ok(patches) + } + + /// Re-run the PMP bring-up after a hibernation restore. The boot loader + /// cold-restarts the PMP (image resident in PMS_SRAM, ASC CPU halted), and + /// nothing replays the probe-time sequence, so the firmware stays down and + /// its clients, cpufreq among them, wait for it in vain. The + /// retained `PmpData` Arc, both `DevresIoMem` mappings and the C power-vote + /// bridge all stay; only the volatile session and firmware are rebuilt. + /// The decision between a halted (reinit + boot) and a live (adopt) firmware + /// follows the ASC CPU run bit, as the NVMe ANS restore does. + fn restore(&self) -> Result<()> { + // Reached from a PM notifier, not the bound probe context, so access the + // retained mappings revocably (the device is still bound during resume). + // try_access() holds an RCU read-side lock while its guard lives: keep each + // guard to the MMIO accesses, never across a mutex, an allocation or the + // sleeping RTKit handshake. + let cpu_running = { + let asc = self.asc_mmio.try_access().ok_or(ENODEV)?; + asc.as_view().relaxed().read(ASC_CPU_CONTROL).cpu_run() != 0 + }; + dev_info!(self.dev, "PMP restore: ASC CPU_RUN={}\n", cpu_running as u32); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; + if rtk.is_crashed() { + dev_err!(self.dev, "PMP restore: session crashed, unrecoverable\n"); + return Err(EIO); + } + if cpu_running { + // The firmware never stopped (an aborted hibernation, or a + // restore failure in the boot kernel): nothing to rebuild. + dev_info!(self.dev, "PMP restore: firmware still running, left as is\n"); + return Ok(()); + } + // Halted firmware: detach the stale session so a fresh HELLO/EPMAP + // boot is accepted, as the NVMe ANS restore reinits its session. + rtk.reinit()?; + } + // The runtime handshake on the PMP endpoint fails against the state + // left from before the hibernation, so clear it first. + *self.state.lock() = PmpState::new()?; + *self.power_command.lock() = Command::Idle; + // Re-patch the SRAM boot args (the cold-restarted firmware re-reads them). + let patches = self.build_patches()?; + { + let pmp = self.pmp_mmio.try_access().ok_or(ENODEV)?; + self.patch_bootargs(&pmp.as_view(), &patches, self.cfg.check_bootargs_uuid)?; + } + // Release the ASC CPU and drive the wake/boot handshake, as probe does. + { + let asc = self.asc_mmio.try_access().ok_or(ENODEV)?; + asc.as_view() + .relaxed() + .update(ASC_CPU_CONTROL, |r| r.with_const_cpu_run::<1>()); + } + self.start()?; + dev_info!(self.dev, "PMP restore: firmware and endpoint back\n"); + Ok(()) + } fn start(&self) -> Result<()> { let mut guard = self.rtkit.lock(); let mut rtk = guard.as_mut().as_pin_mut().unwrap(); @@ -782,6 +868,28 @@ pub unsafe extern "C" fn apple_pmp_wake(data: *const core::ffi::c_void) -> i32 { } } +/// Rebuild the PMP firmware session after a resume from disk (PM_POST_HIBERNATION). +/// The C power-vote bridge calls this with the retained `PmpData` pointer; the +/// bridge, the Arc and the MMIO mappings all stay — only the firmware session is +/// rebuilt. Returns 0 on success or a negative errno. A failure is logged and the +/// PMP stays down (cpufreq keeps its fallback clamp) rather than aborting resume. +/// +/// # Safety +/// `data` must be the live `PmpData` pointer the bridge was registered with, and +/// the caller must serialize against unregister in `Drop`. +#[export] +pub unsafe extern "C" fn apple_pmp_restore(data: *const core::ffi::c_void) -> i32 { + // SAFETY: the C bridge holds the retained foreign reference until reboot. + let data = unsafe { &*data.cast::() }; + match data.restore() { + Ok(()) => 0, + Err(e) => { + dev_err!(data.dev, "PMP restore failed: {:?}\n", e); + e.to_errno() + } + } +} + struct NoBuffer; impl rtkit::Buffer for NoBuffer { fn iova(&self) -> Result { @@ -864,11 +972,16 @@ const HW_CFG_T8140: PmpHwConfig = PmpHwConfig { /// T6041: the DART window starts at 1 TiB; the boot argument header is /// version 5. The firmware advertises endpoint 7 and does not acknowledge /// the IOP power state (-ETIME) until it is started and has its buffer. +/// SRAM kept across a system sleep: the firmware's __DATA, as the +/// ADT's segment-ranges place it (0x500761000, 0x7d000 bytes, after __TEXT at +/// 0x500700000, 0x61000 bytes), the segment T8140's range covers too. +/// Without it the PMP does not answer its endpoint discovery after a sleep +/// that powered the SoC down. const HW_CFG_T6041: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), check_bootargs_uuid: true, coreanalytics: Ep7::Always, - sram_backup: None, + sram_backup: Some((0x61000, 0x7d000)), }; /// T8142: the DART window starts at 1 TiB as on T8140. Its registry-v2 @@ -924,7 +1037,8 @@ impl platform::Driver for PmpDriver { type Data<'bound> = PmpDriver; fn suppress_bind_attrs() -> bool { - // Retained J713 power votes have no teardown path, and an unbound T8140 PMP still runs. + // Retained J713 power votes have no teardown path, and an unbound T8140 or T6040 PMP + // still runs. // SAFETY: read-only machine identity checks; other machines remain unchanged. unsafe { bindings::apple_pmp_keep_bound() } } diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 8c518ef4349e9..8c3d6d2f313d9 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -48,6 +48,12 @@ struct apple_rtkit_helper { bool idle_pending; /* Ends a sleep whose .prepare failed, which gets no .complete. */ struct notifier_block pm_nb; + /* Boot kernel of a resume from disk: RX held for the image. */ + bool rx_held; + /* This restore took over the session the boot kernel started. */ + bool readopted; + /* This restore found the co-processor of our own session halted. */ + bool halted_underneath; void __iomem *asc_base; @@ -63,6 +69,22 @@ bool apple_rtkit_helper_preserves_input(struct device *dev) } EXPORT_SYMBOL_GPL(apple_rtkit_helper_preserves_input); +bool apple_rtkit_helper_readopted(struct device *dev) +{ + struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + + return helper && helper->readopted; +} +EXPORT_SYMBOL_GPL(apple_rtkit_helper_readopted); + +bool apple_rtkit_helper_halted_underneath(struct device *dev) +{ + struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + + return helper && helper->halted_underneath; +} +EXPORT_SYMBOL_GPL(apple_rtkit_helper_halted_underneath); + static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) { struct apple_rtkit_helper *helper = cookie; @@ -119,9 +141,19 @@ static const struct apple_rtkit_ops apple_rtkit_helper_ops = { .shmem_destroy = apple_rtkit_helper_shmem_destroy, }; -static void apple_rtkit_helper_release(void *rtk) +/* Whether the coprocessor's CPU runs: halted, nothing answers on its mailbox. */ +static bool apple_rtkit_helper_cpu_running(struct apple_rtkit_helper *helper) +{ + return readl_relaxed(helper->asc_base + APPLE_ASC_CPU_CONTROL) & + APPLE_ASC_CPU_CONTROL_RUN; +} + +static void apple_rtkit_helper_release(void *data) { - apple_rtkit_free_retaining_buffers(rtk); + struct apple_rtkit_helper *helper = data; + + /* The current session object: a hibernation restore may have replaced it. */ + apple_rtkit_free_retaining_buffers(helper->rtk); } /* @@ -147,8 +179,7 @@ static int apple_rtkit_helper_live(struct apple_rtkit_helper *helper) return -EINVAL; } - if (!(readl_relaxed(helper->asc_base + APPLE_ASC_CPU_CONTROL) & - APPLE_ASC_CPU_CONTROL_RUN)) { + if (!apple_rtkit_helper_cpu_running(helper)) { dev_warn(helper->dev, "RTKit handoff without a running CPU\n"); return -ENODEV; } @@ -246,6 +277,21 @@ static int apple_rtkit_helper_inherit_buffers( return count; } +/* + * Whether an adopted session refuses buffers its inventory does not list. With + * an inventory, and on T8132 and the J614s without one, no new allocation is + * safe: the firmware still uses what it negotiated with its first host. + * Producers that predate the inventory (T8140) keep mapping IOVAs inside the + * SRAM region. Probe and the restore after a hibernation adopt by this rule. + */ +static bool apple_rtkit_helper_strict(struct device *dev, int count) +{ + return count > 0 || + of_device_is_compatible(dev->of_node, "apple,t8132-mtp") || + (of_machine_is_compatible("apple,j614s") && + of_device_is_compatible(dev->of_node, "apple,t6041-mtp")); +} + static int apple_rtkit_helper_pm_register(struct apple_rtkit_helper *helper); static int apple_rtkit_helper_probe(struct platform_device *pdev) @@ -330,16 +376,9 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) } if (live) { - /* - * Set before RX starts: with an inventory, or on T8132, no new - * buffer allocation is safe here. Producers that predate the - * inventory (T8140) keep mapping IOVAs inside the SRAM region. - */ + /* Set before RX starts, which may bring a buffer request. */ helper->adopted = true; - helper->strict_buffers = count > 0 || - of_device_is_compatible(dev->of_node, "apple,t8132-mtp") || - (of_machine_is_compatible("apple,j614s") && - of_device_is_compatible(dev->of_node, "apple,t6041-mtp")); + helper->strict_buffers = apple_rtkit_helper_strict(dev, count); helper->rtk = apple_rtkit_init_adopted_with_buffers( dev, helper, NULL, 0, &apple_rtkit_helper_ops, buffers, count); if (IS_ERR(helper->rtk)) @@ -347,7 +386,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) "Failed to adopt RTKit\n"); ret = devm_add_action_or_reset(dev, apple_rtkit_helper_release, - helper->rtk); + helper); if (ret) return ret; ret = apple_rtkit_helper_pm_register(helper); @@ -388,6 +427,17 @@ static void apple_rtkit_helper_remove(struct platform_device *pdev) if (helper->adopted || helper->preserved) return; + /* + * After a resume from disk the boot loader left the coprocessor halted + * with fresh firmware loaded, while the restored host session still + * believes it runs: nothing would answer the quiesce. Drop the session + * without a handshake so that the next probe starts the CPU again. + */ + if (!apple_rtkit_helper_cpu_running(helper)) { + dev_info(&pdev->dev, "co-processor halted underneath (resume from disk): removed without quiesce\n"); + return; + } + if (apple_rtkit_is_running(helper->rtk)) apple_rtkit_quiesce(helper->rtk); @@ -554,12 +604,194 @@ static int __maybe_unused apple_rtkit_helper_resume(struct device *dev) return 0; } -static int __maybe_unused apple_rtkit_helper_freeze(struct device *dev) +/* + * What the boot kernel of a resume from disk leaves for the restored kernel + * about a session it started itself: the shared buffers the firmware holds. + */ +struct apple_rtkit_helper_hib { + u32 count; + u32 reserved; + struct apple_rtkit_buffer_desc buf[APPLE_RTKIT_BUF_COUNT]; +}; +static_assert(sizeof(struct apple_rtkit_helper_hib) <= APPLE_RTKIT_HIB_HANDOFF_MAX); + +static void apple_rtkit_helper_hib_tag(struct device *dev, char *tag, size_t len) +{ + snprintf(tag, len, "rtkh:%s", dev_name(dev)); +} + +/* + * Hibernation of an adopted session. The image is written while the firmware + * keeps running; its buffers live in SRAM or in no-map reserved RAM, so they + * are not part of the image and the host side has nothing to quiesce. The + * restore re-adopts the instance the boot loader started for the resume boot, + * exactly as probe adopted the first one. + * + * A session this driver started itself is left running as well: the MTP + * firmware does not come back from QUIESCED (it does not answer the endpoint + * discovery of the next start). When this is the boot kernel of a resume from + * disk (it started MTP so that the disk passphrase can be typed), it hands + * the running firmware to the image: mailbox RX is held, so that what the + * firmware sends now stays queued for the restored kernel, the messages + * already received are handled, and the shared buffers are described for the + * restored kernel, which can only take the session over if its own buffers + * sit at the same IOVAs (the firmware keeps writing to the IOVAs it was + * given, and after the restore the DART translates them through the image's + * page tables). + */ +static int apple_rtkit_helper_freeze(struct device *dev) { struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + struct apple_rtkit_helper_hib rec = {}; + char tag[32]; + int n, ret; + + helper->readopted = false; + helper->halted_underneath = false; + if (helper->adopted || !apple_rtkit_hib_restoring()) + return 0; + if (!apple_rtkit_helper_cpu_running(helper) || + !apple_rtkit_is_running(helper->rtk)) + return 0; + + apple_rtkit_hold_rx(helper->rtk); + apple_rtkit_flush_rx(helper->rtk); + helper->rx_held = true; - /* A hibernation image cannot recreate the firmware's live DMA ownership. */ - return helper->adopted ? -EBUSY : 0; + n = apple_rtkit_describe_buffers(helper->rtk, rec.buf, ARRAY_SIZE(rec.buf)); + if (n < 0) { + dev_warn(dev, "resume from disk: cannot describe the RTKit buffers: %d\n", n); + return 0; + } + rec.count = n; + apple_rtkit_helper_hib_tag(dev, tag, sizeof(tag)); + ret = apple_rtkit_hib_handoff_put(tag, &rec, sizeof(rec)); + if (ret) + dev_warn(dev, "resume from disk: no hand-off record for the image: %d\n", ret); + else + dev_dbg(dev, "resume from disk: RTKit session left running for the image (%d buffers)\n", + n); + return 0; +} + +static int apple_rtkit_helper_thaw(struct device *dev) +{ + struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + char tag[32]; + + /* The jump into the image failed: this kernel keeps the session. */ + if (helper->rx_held) { + helper->rx_held = false; + apple_rtkit_helper_hib_tag(dev, tag, sizeof(tag)); + apple_rtkit_hib_handoff_drop(tag); + apple_rtkit_resume_rx(helper->rtk); + } + return 0; +} + +/* + * A session this driver started itself, after a resume from disk. Taken over + * in place when the boot kernel left it running and its buffers match ours; + * otherwise left as before (stale until the driver is reloaded). + */ +static int apple_rtkit_helper_restore_own(struct apple_rtkit_helper *helper) +{ + struct device *dev = helper->dev; + struct apple_rtkit_helper_hib rec = {}, own = {}; + char tag[32]; + int i, n, ret; + + apple_rtkit_helper_hib_tag(dev, tag, sizeof(tag)); + ret = apple_rtkit_hib_handoff_take(tag, &rec, sizeof(rec)); + if (ret < 0) { + if (apple_rtkit_helper_cpu_running(helper)) { + dev_warn(dev, "resume from disk: co-processor runs without a hand-off from the boot kernel (%d): not taken over\n", + ret); + } else { + helper->halted_underneath = true; + dev_info(dev, "resume from disk: self-started RTKit session left for a reload\n"); + } + return 0; + } + if (!apple_rtkit_helper_cpu_running(helper)) { + helper->halted_underneath = true; + dev_warn(dev, "resume from disk: hand-off present but the co-processor is halted: not taken over\n"); + return 0; + } + + n = apple_rtkit_describe_buffers(helper->rtk, own.buf, ARRAY_SIZE(own.buf)); + own.count = n < 0 ? 0 : n; + if (n < 0 || rec.count > ARRAY_SIZE(rec.buf) || own.count != rec.count || + memcmp(own.buf, rec.buf, rec.count * sizeof(rec.buf[0]))) { + dev_err(dev, "resume from disk: RTKit buffers of the boot kernel differ from the image's: not taken over\n"); + for (i = 0; i < min_t(u32, rec.count, ARRAY_SIZE(rec.buf)); i++) + dev_err(dev, " boot kind %u iova %#llx size %#llx\n", + rec.buf[i].kind, rec.buf[i].iova, rec.buf[i].size); + for (i = 0; i < own.count; i++) + dev_err(dev, " image kind %u iova %#llx size %#llx\n", + own.buf[i].kind, own.buf[i].iova, own.buf[i].size); + return 0; + } + + ret = apple_rtkit_readopt(helper->rtk); + if (ret) { + dev_err(dev, "resume from disk: RTKit session not taken over: %d\n", ret); + return 0; + } + helper->readopted = true; + dev_dbg(dev, "resume from disk: took over the RTKit session the boot kernel started (%d buffers at the same IOVAs)\n", + n); + return 0; +} + +static int apple_rtkit_helper_restore(struct device *dev) +{ + struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + struct apple_rtkit_inherited_buffer buffers[APPLE_RTKIT_HELPER_MAX_BUFFERS] = {}; + struct apple_rtkit *old, *rtk; + int count, live; + + if (!helper->adopted) + return apple_rtkit_helper_restore_own(helper); + + /* + * The restored tree still carries the handoff the image's boot stage + * wrote. The resume boot's loader hands the same firmware over at the + * same SRAM and no-map addresses (the inventory comes from the firmware + * carveouts, not from the kernel allocator), so re-read it and check + * that a CPU is running behind it. + */ + live = apple_rtkit_helper_live(helper); + if (live < 0) { + dev_err(dev, "hibernation restore: invalid live RTKit handoff: %d\n", live); + return live; + } + if (!live) { + dev_err(dev, "hibernation restore: no live RTKit session to re-adopt\n"); + return -ENODEV; + } + count = apple_rtkit_helper_inherit_buffers(helper, buffers); + if (count < 0) { + dev_err(dev, "hibernation restore: invalid inherited RTKit buffers: %d\n", count); + return count; + } + + /* As at probe, before the new session's RX starts. */ + helper->strict_buffers = apple_rtkit_helper_strict(dev, count); + rtk = apple_rtkit_init_adopted_with_buffers(dev, helper, NULL, 0, + &apple_rtkit_helper_ops, buffers, count); + if (IS_ERR(rtk)) { + dev_err(dev, "hibernation restore: failed to re-adopt RTKit: %pe\n", rtk); + return PTR_ERR(rtk); + } + + /* The old object described an instance that no longer exists. */ + old = helper->rtk; + helper->rtk = rtk; + apple_rtkit_free_retaining_buffers(old); + dev_dbg(dev, "re-adopted the running RTKit session after hibernation (%d retained buffers)\n", + count); + return 0; } static const struct dev_pm_ops apple_rtkit_helper_pm_ops = { @@ -569,6 +801,8 @@ static const struct dev_pm_ops apple_rtkit_helper_pm_ops = { .resume = pm_sleep_ptr(apple_rtkit_helper_resume), .freeze = pm_sleep_ptr(apple_rtkit_helper_freeze), .poweroff = pm_sleep_ptr(apple_rtkit_helper_freeze), + .thaw = pm_sleep_ptr(apple_rtkit_helper_thaw), + .restore = pm_sleep_ptr(apple_rtkit_helper_restore), }; static const struct of_device_id apple_rtkit_helper_of_match[] = { diff --git a/drivers/soc/apple/rtkit-hib-handoff.c b/drivers/soc/apple/rtkit-hib-handoff.c new file mode 100644 index 0000000000000..0700ab8488284 --- /dev/null +++ b/drivers/soc/apple/rtkit-hib-handoff.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple RTKit: hand a running co-processor session from the kernel that loads + * a hibernation image (the boot kernel) to the kernel restored from it. + * + * A co-processor that the boot kernel started keeps running across the jump + * into the image, but everything the boot kernel knew about it is overwritten + * by the image. The few facts the restored kernel needs to take the session + * over (where the firmware's shared buffers are, how far a receive ring got) + * are left in __nosavedata: the image never saves those pages, so after the + * restore they still hold what the boot kernel wrote, provided both kernels + * are the same build loaded at the same physical address. Each record carries + * its own physical address and a checksum, so a record from a kernel loaded + * elsewhere, or no record at all, reads as absent. + * + * The storage must be in the kernel image, so this only works with RTKit + * built in. A taken record is cleared. + */ + +#include +#include +#include +#include +#include + +#if IS_BUILTIN(CONFIG_APPLE_RTKIT) && IS_ENABLED(CONFIG_HIBERNATION) + +#define HANDOFF_MAGIC 0x52544b48 /* "RTKH" */ +#define HANDOFF_VERSION 1 +#define HANDOFF_SLOTS 4 +#define HANDOFF_TAG 32 + +struct apple_rtkit_hib_slot { + u32 magic; + u32 version; + u64 self_phys; + char tag[HANDOFF_TAG]; + u32 len; + u32 sum; + u8 data[APPLE_RTKIT_HIB_HANDOFF_MAX]; +}; + +static struct apple_rtkit_hib_slot handoff[HANDOFF_SLOTS] __nosavedata; +static DEFINE_SPINLOCK(handoff_lock); + +/* FNV-1a over everything but the checksum itself. */ +static u32 handoff_sum(const struct apple_rtkit_hib_slot *s) +{ + const u8 *p = (const u8 *)s; + size_t off = offsetof(struct apple_rtkit_hib_slot, sum); + u32 h = 0x811c9dc5; + size_t i; + + for (i = 0; i < sizeof(*s); i++) { + if (i >= off && i < off + sizeof(s->sum)) + continue; + h ^= p[i]; + h *= 0x01000193; + } + return h; +} + +static bool handoff_valid(const struct apple_rtkit_hib_slot *s) +{ + return s->magic == HANDOFF_MAGIC && s->version == HANDOFF_VERSION && + s->self_phys == __pa_symbol(s) && + s->len <= APPLE_RTKIT_HIB_HANDOFF_MAX && + s->tag[HANDOFF_TAG - 1] == '\0' && s->sum == handoff_sum(s); +} + +static struct apple_rtkit_hib_slot *handoff_find(const char *tag) +{ + int i; + + for (i = 0; i < HANDOFF_SLOTS; i++) + if (handoff_valid(&handoff[i]) && !strcmp(handoff[i].tag, tag)) + return &handoff[i]; + return NULL; +} + +bool apple_rtkit_hib_restoring(void) +{ + return pm_hibernate_is_loading(); +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_restoring); + +int apple_rtkit_hib_handoff_put(const char *tag, const void *data, size_t len) +{ + struct apple_rtkit_hib_slot *s; + unsigned long flags; + int i, ret = 0; + + if (len > APPLE_RTKIT_HIB_HANDOFF_MAX || strlen(tag) >= HANDOFF_TAG) + return -EINVAL; + + spin_lock_irqsave(&handoff_lock, flags); + s = handoff_find(tag); + for (i = 0; !s && i < HANDOFF_SLOTS; i++) + if (!handoff_valid(&handoff[i])) + s = &handoff[i]; + if (s) { + memset(s, 0, sizeof(*s)); + s->magic = HANDOFF_MAGIC; + s->version = HANDOFF_VERSION; + s->self_phys = __pa_symbol(s); + strscpy(s->tag, tag, sizeof(s->tag)); + s->len = len; + memcpy(s->data, data, len); + s->sum = handoff_sum(s); + } else { + ret = -ENOSPC; + } + spin_unlock_irqrestore(&handoff_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_put); + +int apple_rtkit_hib_handoff_take(const char *tag, void *data, size_t len) +{ + struct apple_rtkit_hib_slot *s; + unsigned long flags; + int ret; + + spin_lock_irqsave(&handoff_lock, flags); + s = handoff_find(tag); + if (!s) { + ret = -ENOENT; + } else if (s->len != len) { + ret = -EINVAL; + } else { + memcpy(data, s->data, len); + ret = len; + } + if (s) + memset(s, 0, sizeof(*s)); + spin_unlock_irqrestore(&handoff_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_take); + +void apple_rtkit_hib_handoff_drop(const char *tag) +{ + struct apple_rtkit_hib_slot *s; + unsigned long flags; + + spin_lock_irqsave(&handoff_lock, flags); + s = handoff_find(tag); + if (s) + memset(s, 0, sizeof(*s)); + spin_unlock_irqrestore(&handoff_lock, flags); +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_drop); + +#else + +bool apple_rtkit_hib_restoring(void) +{ + return false; +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_restoring); + +int apple_rtkit_hib_handoff_put(const char *tag, const void *data, size_t len) +{ + return -EOPNOTSUPP; +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_put); + +int apple_rtkit_hib_handoff_take(const char *tag, void *data, size_t len) +{ + return -ENOENT; +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_take); + +void apple_rtkit_hib_handoff_drop(const char *tag) +{ +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_drop); + +#endif diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 8230ebe11d411..1f7b101d5d195 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1641,6 +1641,63 @@ bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep) } EXPORT_SYMBOL_GPL(apple_rtkit_has_endpoint); +int apple_rtkit_describe_buffers(struct apple_rtkit *rtk, + struct apple_rtkit_buffer_desc *desc, + unsigned int max) +{ + const struct apple_rtkit_shmem *bufs[APPLE_RTKIT_BUF_COUNT] = { + [APPLE_RTKIT_BUF_IOREPORT] = &rtk->ioreport_buffer, + [APPLE_RTKIT_BUF_COREANALYTICS] = &rtk->coreanalytics_buffer, + [APPLE_RTKIT_BUF_CRASHLOG] = &rtk->crashlog_buffer, + [APPLE_RTKIT_BUF_OSLOG] = &rtk->oslog_buffer, + [APPLE_RTKIT_BUF_KDEBUG0] = &rtk->kdebug_buffer0, + [APPLE_RTKIT_BUF_KDEBUG1] = &rtk->kdebug_buffer1, + [APPLE_RTKIT_BUF_PMP_AUX] = &rtk->pmp_aux_buffer, + [APPLE_RTKIT_BUF_SYSLOG] = &rtk->syslog_buffer, + }; + unsigned int i, n = 0; + + for (i = 0; i < APPLE_RTKIT_BUF_COUNT; i++) { + if (!bufs[i]->size) + continue; + if (n == max) + return -ENOSPC; + desc[n].kind = i; + desc[n].is_mapped = bufs[i]->is_mapped; + desc[n].iova = bufs[i]->iova; + desc[n].size = bufs[i]->size; + n++; + } + return n; +} +EXPORT_SYMBOL_GPL(apple_rtkit_describe_buffers); + +int apple_rtkit_readopt(struct apple_rtkit *rtk) +{ + /* + * Only a session that was running when it was left can be taken over: + * its endpoint map, protocol version, syslog geometry and buffers then + * describe the same firmware build and the same IOVAs as the instance + * that runs now (the caller compared the buffers). + */ + if (READ_ONCE(rtk->shutting_down) || !apple_rtkit_is_running(rtk)) + return -EINVAL; + + apple_mbox_hold_rx(rtk->mbox); + /* Acknowledgments still queued belong to the instance that is gone. */ + cancel_work_sync(&rtk->sync_ack_work); + flush_workqueue(rtk->wq); + kfifo_reset(&rtk->sync_ack_fifo); + + reinit_completion(&rtk->iop_pwr_ack_completion); + reinit_completion(&rtk->ap_pwr_ack_completion); + + /* Messages the other kernel left queued in the mailbox arrive now. */ + apple_mbox_release_rx(rtk->mbox); + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_readopt); + int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) { u64 msg; @@ -2146,8 +2203,9 @@ static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk, return 0; } -static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, - unsigned int state) +/* Ask the IOP for a power state and wait for its acknowledgment; the caller judges the state. */ +static int apple_rtkit_send_iop_power_state(struct apple_rtkit *rtk, + unsigned int state) { u64 msg; int ret; @@ -2160,7 +2218,15 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, if (ret) return ret; - ret = apple_rtkit_wait(rtk, &rtk->iop_pwr_ack_completion); + return apple_rtkit_wait(rtk, &rtk->iop_pwr_ack_completion); +} + +static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, + unsigned int state) +{ + int ret; + + ret = apple_rtkit_send_iop_power_state(rtk, state); if (ret) return ret; @@ -2298,11 +2364,16 @@ int apple_rtkit_idle_with(struct apple_rtkit *rtk, void (*after_ap)(void *data), if (after_ap) after_ap(data); - ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); + ret = apple_rtkit_send_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); if (ret) { dev_err(rtk->dev, "RTKit: IOP idle not acknowledged: %d\n", ret); return ret; } + if (rtk->iop_power_state != APPLE_RTKIT_PWR_STATE_IDLE) { + dev_dbg(rtk->dev, "RTKit: IOP acknowledged %#x instead of idle\n", + rtk->iop_power_state); + return APPLE_RTKIT_IDLE_RETAINED; + } rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE; @@ -2514,18 +2585,21 @@ static int apple_rtkit_polled_state(struct apple_rtkit *rtk, u8 type, unsigned i return -ETIMEDOUT; } -int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, - unsigned int *us) +/* The receive path while a notice is polled for, and the usual one after it. */ +static void apple_rtkit_polled_set_rx(struct apple_rtkit *rtk, + void (*rx)(struct apple_mbox *mbox, + struct apple_mbox_msg msg, void *cookie)) { unsigned long flags; - u64 owed; - u32 status; - int ret; - *us = 0; spin_lock_irqsave(&rtk->mbox->rx_lock, flags); - rtk->mbox->rx = apple_rtkit_polled_rx; + rtk->mbox->rx = rx; spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); +} + +static void apple_rtkit_polled_owed(struct apple_rtkit *rtk) +{ + u64 owed; /* What's already waiting is answered first, or the firmware may not go idle. */ apple_mbox_poll(rtk->mbox); @@ -2537,6 +2611,17 @@ int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_con if (!apple_rtkit_polled_send(rtk, echo)) dev_err(rtk->dev, "RTKit: oslog echo failed before the idle\n"); } +} + +int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, + unsigned int *us) +{ + u32 status; + int ret; + + *us = 0; + apple_rtkit_polled_set_rx(rtk, apple_rtkit_polled_rx); + apple_rtkit_polled_owed(rtk); ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, &rtk->ap_power_state, us); @@ -2549,14 +2634,43 @@ int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_con APPLE_ASC_CPU_STATUS_DOWN, APPLE_RTKIT_POLLED_STEP_US, 5000); - spin_lock_irqsave(&rtk->mbox->rx_lock, flags); - rtk->mbox->rx = apple_rtkit_rx; - spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + apple_rtkit_polled_set_rx(rtk, apple_rtkit_rx); return ret; } EXPORT_SYMBOL_GPL(apple_rtkit_sleep_idle_polled); +/* + * For a co-processor that stays up: its client may still be owed a reply or an event on + * its own endpoints. Those go the usual way, to recv_message_early and, if it leaves + * them, to the receive work, which runs again after the wake. + */ +static void apple_rtkit_ap_polled_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, + void *cookie) +{ + if (msg.msg1 >= APPLE_RTKIT_APP_ENDPOINT_START) + apple_rtkit_rx(mbox, msg, cookie); + else + apple_rtkit_polled_rx(mbox, msg, cookie); +} + +int apple_rtkit_ap_idle_polled(struct apple_rtkit *rtk, unsigned int *us) +{ + int ret; + + *us = 0; + apple_rtkit_polled_set_rx(rtk, apple_rtkit_ap_polled_rx); + apple_rtkit_polled_owed(rtk); + + ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + &rtk->ap_power_state, us); + + apple_rtkit_polled_set_rx(rtk, apple_rtkit_rx); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_ap_idle_polled); + void apple_rtkit_mailbox_recv_off(struct apple_rtkit *rtk) { apple_mbox_set_recv_enabled(rtk->mbox, false); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 02fcc78b25564..4c12380cdcec4 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -971,6 +971,18 @@ struct SepData { shutting_down: Atomic, capture_epoch: Atomic, + // A hibernation is under way (prepare until the kernel knows how it + // ended): requests Linux starts get EAGAIN and the enclave's xART writes + // are refused, so that neither the host state nor the gigalocker changes + // behind the image. Separate from `sleep_gate`, which only a + // firmware-driven suspend moves. See pm.rs. + session_held: Atomic, + // The enclave was booted anew under a hibernation image (a resume from + // disk on a cold boot): nothing of this session may reach it any more. + // Set once; the device is then probed again. + session_gone: Atomic, + xart_held_logged: Atomic, + // The system is going to sleep: no capture may start. Changed under the // bio session lock; `sleep_over` is signalled when it clears. suspending: Atomic, @@ -1299,6 +1311,9 @@ impl SepData { registered: Atomic::new(false), shutting_down: Atomic::new(false), capture_epoch: Atomic::new(0), + session_held: Atomic::new(false), + session_gone: Atomic::new(false), + xart_held_logged: Atomic::new(false), suspending: Atomic::new(false), sleep_over <- new_condvar!("SepData::sleep_over"), captures_running: Atomic::new(0), @@ -1501,24 +1516,29 @@ impl SepData { } } + /// Sends a request Linux starts. Refused while a hibernation holds the + /// session (EAGAIN) and once the enclave has been booted anew under it + /// (ENODEV). fn send(&self, msg: Message) -> Result<()> { - if self.shutting_down.load(Relaxed) { + if self.shutting_down.load(Relaxed) || self.session_gone.load(Relaxed) { return Err(ENODEV); } // Checked with the mailbox lock held: a sender can wait for it for longer - // than the system takes to tell the enclave it's going to sleep. + // than the system takes to tell the enclave it's going to sleep. The + // hibernation hold is taken under it too. let mbox = self.mbox.lock(); let mbox = mbox.as_ref().ok_or(ENODEV)?; - if !self.sleep_gate().may_request() { + if !self.sleep_gate().may_request() || self.session_held.load(Relaxed) { return Err(EAGAIN); } Self::ring(mbox, msg) } /// As [`Self::send`], for an answer to something the enclave asked: it - /// waits for these before it can answer the sleep request. + /// waits for these before it can answer the sleep request. A hibernation + /// hold does not apply to them, but a lost session does. fn send_reply(&self, msg: Message) -> Result<()> { - if self.shutting_down.load(Relaxed) { + if self.shutting_down.load(Relaxed) || self.session_gone.load(Relaxed) { return Err(ENODEV); } let mbox = self.mbox.lock(); @@ -1529,6 +1549,45 @@ impl SepData { Self::ring(mbox, msg) } + /// A hibernation starts (`held`) or ended without a cold boot. The hold + /// is taken after the capture and entropy holds and lifted before them. + pub(crate) fn hibernation_hold(&self, held: bool) { + { + let _mbox = self.mbox.lock(); + self.session_held.store(held, Relaxed); + } + if held { + dev_info_ratelimited!(self.dev, "hibernation: holding SEP requests and xART writes\n"); + } else { + self.xart_held_logged.store(false, Relaxed); + dev_info_ratelimited!( + self.dev, + "hibernation: ended without a cold boot, SEP session kept\n" + ); + } + } + + /// Whether this kernel registered with the enclave (a kernel about to + /// load a hibernation image must not have). + pub(crate) fn attached(&self) -> bool { + self.registered.load(Relaxed) && !self.session_gone.load(Relaxed) + } + + /// The boot loader booted the enclave anew under the hibernation image. + /// Everything this session holds there is gone: refuse every message, so + /// that the fresh sepOS sees nothing before the new registration, and let + /// `pm_shim.c` probe the device again. Returns whether to probe again. + pub(crate) fn lose_session(&self) -> bool { + if self.session_gone.xchg(true, Relaxed) { + return false; + } + dev_info_ratelimited!( + self.dev, + "resume from disk on a cold boot: the boot loader restarted the enclave; dropping this session and probing again\n" + ); + true + } + pub(crate) fn sleep_gate(&self) -> pm::SleepGate { pm::SleepGate::from_u32(self.sleep_gate.load(Acquire)) } @@ -1548,6 +1607,15 @@ impl SepData { pm::SleepGate::Open => {} _ => return Err(EBUSY), } + // Booted anew under a hibernation image: nothing of the old session may + // reach it, the sleep request included (see `lose_session`). + if self.session_gone.load(Relaxed) { + dev_warn_ratelimited!( + self.dev, + "the enclave was restarted under a resume from disk, so it isn't told of this sleep\n" + ); + return Ok(()); + } // Do not commit a running enclave whose sleep protocol is not ready. if self.phase.load(Relaxed) != PHASE_READY || !self.endpoint_present(proto::EP_CONTROL) { dev_warn_ratelimited!( @@ -1875,12 +1943,16 @@ impl SepData { /// way to sleep waits for it. It must not then write its request into /// the suspend. Lock order: mailbox, then control. fn send_entropy_request(&self, msg: Message) -> Result<()> { - if self.shutting_down.load(Relaxed) { + if self.shutting_down.load(Relaxed) || self.session_gone.load(Relaxed) { return Err(ENODEV); } let mbox = self.mbox.lock(); let mbox = mbox.as_ref().ok_or(ENODEV)?; - if !self.sleep_gate().may_request() || !self.control.lock().entropy_may_send() { + // Under the mailbox lock, as in `send`. + if !self.sleep_gate().may_request() + || self.session_held.load(Relaxed) + || !self.control.lock().entropy_may_send() + { return Err(EAGAIN); } Self::ring(mbox, msg) @@ -2158,6 +2230,21 @@ impl SepData { } fn service_xarm(&self, req: &crate::xarm::XarmRequest) { + // Behind a hibernation image nothing may change: the restored kernel + // would hold a stale view of the gigalocker and of the host files. + // Refused like a read-only gigalocker; reads are still served. + if xarm::is_write(req.opcode) && self.session_held.load(Relaxed) { + if !self.xart_held_logged.xchg(true, Relaxed) { + dev_warn_ratelimited!( + self.dev, + "xART: refused the enclave's write (op 0x{:02x}) during a hibernation\n", + req.opcode + ); + } + self.fail_xarm(req.tag); + return; + } + let want = req.length as usize; if want > OOL_SIZE_XARM { self.fail_xarm(req.tag); @@ -3639,7 +3726,6 @@ impl Drop for SepDriver { } } - #[pin_data] struct SepModule { // field-init order enables the DT nodes before the driver registers; a failure here aborts module init diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 938e9b9a529a7..f32945e01f5df 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -26,7 +26,14 @@ void sep_cancel_delayed_work_sync(void *work); /* -- pm_shim.c ---------------------------------------------------------- */ -int sep_pm_register(void *dev, void (*event)(bool entering), int (*sleep)(void), +/* Sleep events for sep_pm_register()'s callback; mirrored in pm.rs. */ +#define SEP_PM_SLEEP 0 +#define SEP_PM_WAKE 1 +#define SEP_PM_HIBERNATE 2 +#define SEP_PM_HIBERNATE_WARM 3 +#define SEP_PM_HIBERNATE_COLD 4 +#define SEP_PM_RESTORE 5 +int sep_pm_register(void *dev, int (*event)(int event), int (*sleep)(void), void (*wake)(void), bool (*told)(void)); void sep_pm_unregister(void); int sep_pm_keep_domains(void *dev); diff --git a/drivers/soc/apple/sleep-internal.h b/drivers/soc/apple/sleep-internal.h new file mode 100644 index 0000000000000..c5286dbcb088e --- /dev/null +++ b/drivers/soc/apple/sleep-internal.h @@ -0,0 +1,332 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * The coprocessor and enclave registry, and what the Apple SoC sleep driver + * (sleep.c) shares with a SoC's own part of it (sleep-t6040.c for T6040). + * Drivers register through ; the controller bound on + * this SoC walks the table at its commit. + */ +#ifndef _APPLE_SLEEP_INTERNAL_H_ +#define _APPLE_SLEEP_INTERNAL_H_ + +#include +#include +#include +#include +#include +#include + +#include + +struct device; +struct device_node; +struct platform_device; +struct seq_file; + +/* The coprocessors the last mile stops. Each driver idles its own first. */ +#define APPLE_SLEEP_COPROCS 6 + +struct apple_sleep_coproc { + struct device *dev; + void __iomem *asc; + bool pcie_ans; + int (*last_check)(void *data); + int (*last_stop)(void *data); + void *data; + bool ready; + bool stopped; + u32 status; + int stop_ret; + /* No ASC: its last stop takes it whole, power word @ps included, before the others. */ + bool first; + unsigned int order; /* among those registered first, lowest runs first */ + u32 ps; +}; + +/* Irreversible enclave stop; nonzero is legal only if SLEEP was not sent. */ +struct apple_sleep_enclave { + struct device *dev; + int (*stop)(void *data); + void *data; +}; + +extern struct apple_sleep_coproc apple_sleep_coprocs[APPLE_SLEEP_COPROCS]; +extern struct apple_sleep_enclave apple_sleep_enclave; +extern struct mutex apple_sleep_coprocs_lock; + +struct apple_sleep_coproc *apple_sleep_coproc_find(struct device *dev); + +/* + * The sleep controller bound on this SoC. Each binds only to its own SoC's + * compatible, so at most one is bound. Bound, it tells the registry when its + * probe is complete: from then on a suspend through firmware that is not a dry + * run is one whose commit stops the registered coprocessors. + * + * APPLE_SLEEP_REG_ENCLAVE_ANY: take every enclave registration. Without it one + * is taken only while such a commit is the case. + * APPLE_SLEEP_REG_LID_AT_WAKE, APPLE_SLEEP_REG_USB_WAKE_PORTS: the sleep takes the lid's state + * at the wake, and the USB wake ports. Without them + * apple_sleep_lid_closed_at_wake() and apple_sleep_usb_port_can_wake() answer as they do + * with no sleep driver, whatever the SMC's and the USB host's drivers told. + */ +#define APPLE_SLEEP_REG_ENCLAVE_ANY BIT(0) +#define APPLE_SLEEP_REG_LID_AT_WAKE BIT(1) +#define APPLE_SLEEP_REG_USB_WAKE_PORTS BIT(2) + +int apple_sleep_registry_bind(unsigned int flags); +void apple_sleep_registry_unbind(void); +void apple_sleep_registry_ready(void); + +/* Must match aura's struct aura_s2r_mailbox. */ +#define AURA_S2R_MAGIC 0x3152325341525541ULL /* "AURAS2R1" */ + +struct aura_s2r_mailbox { + u64 magic; /* aura, before the kernel starts */ + u64 boot_mpidr; /* aura: the only core that may take resume_pa */ + u64 resume_pa; /* kernel, armed before a commit; aura clears it */ + u64 resumes; /* aura: resume_pa taken this many times */ +}; + +/* + * Registers that are the same on every SoC this driver knows. + */ + +/* PS word */ +#define PS_AUTO_ENABLE BIT(28) +#define PS_MIN GENMASK(19, 16) +#define PS_UNLATCH BIT(12) +#define PS_BUSY BIT(11) +#define PS_DEV_DISABLE BIT(10) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) +#define PS_ACTIVE 0xf +#define PS_OFF_TIMEOUT_US 5000 +#define PS_BUSY_SETTLE_US 1000 + +/* CPU start block, as aura's smp_start_cpu() uses it. A core's bit is the SoC's. */ +#define CPU_START_STOP 0x0 +#define CPU_START_SYS 0x4 +#define CPU_START_CLUSTER(n) (0x8 + (n) * 4) + +/* Watchdog block, ADT /arm-io/wdt reg[0]. WD2 is the sleep safety net. */ +#define WDT_WD0_COUNT 0x00 +#define WDT_WD0_CTL 0x0c +#define WDT_WD1_COUNT 0x10 +#define WDT_WD2_COUNT 0x20 +#define WDT_WD2_ALARM 0x28 +#define WDT_WD2_CTL 0x2c +#define WDT_TICKS_PER_S 24000000 +/* The sleep's own safety net: ADT /arm-io/wdt simple-reconfig-wdog-icc-time. */ +#define WDT_SLEEP_SECS 5 +/* Device suspend and resume can take a while, and nothing else recovers a hang in them. */ +#define WDT_SUSPEND_SECS 120 + +/* The AP queue of the PMU's SPMI controller, and its response vector's enable and ack */ +#define SPMI_STATUS 0x00 +#define SPMI_STATUS_RX_EMPTY BIT(24) +#define SPMI_CMD 0x04 +#define SPMI_REPLY 0x08 +#define SPMI_NOTIFY_ENABLE 0x40 +#define SPMI_NOTIFY_REPLY BIT(0) +#define SPMI_NOTIFY_ACK 0x80 +#define SPMI_CMD_ACTIVE BIT(15) +#define SPMI_OP_EXT_WRITE(n) ((n) - 1) +#define SPMI_OP_EXT_READ(n) (0x20 | ((n) - 1)) +#define SPMI_OP_ZERO_WRITE 0x80 +#define SPMI_OP_SLEEP 0x11 +#define SPMI_OP_WAKEUP 0x13 +#define SPMI_POLL_US 20000 + +/* A coprocessor's ASC */ +#define ASC_CPU_CONTROL 0x44 +#define ASC_CPU_CONTROL_RUN BIT(4) +#define ASC_CPU_STATUS 0x48 + +/* The part of the PCIe block that ANS shares, whether or not its host is bound. */ +#define APCIE_AXI_EN0 0x104 +#define APCIE_AXI_EN1 0x108 +#define APCIE_CLK_CTRL 0x0 +#define APCIE_CLK_BUSY BIT(31) +#define APCIE_CLK_RUN BIT(5) +#define APCIE_CLK_OFF BIT(1) +#define APCIE_CLK_OFF2 BIT(2) +#define APCIE_CLK_START 0x4 +#define APCIE_CLK_START_GO BIT(31) +#define APCIE_PHY_COMMON 0x4000 +#define APCIE_PHY_CTRL 0x8000 +#define APCIE_PHY_CTRL_CLK0REQ BIT(0) +#define APCIE_PHY_CTRL_CLK1REQ BIT(1) +#define APCIE_PHY_CTRL_CLK0ACK BIT(2) +#define APCIE_PHY_CTRL_CLK1ACK BIT(3) +#define APCIE_PHY_CTRL_RESET BIT(4) +#define APCIE_PHY_CTRL_UP BIT(27) +#define APCIE_PHY_RUN 0x8004 +#define APCIE_PHY_RUN_EN BIT(0) +#define APCIE_PHY_RUN_EN2 BIT(4) +#define APCIE_PHY_READY 0x8008 +#define APCIE_PHY_IP_PLL_CTL 0x90 +#define APCIE_PHY_IP_PLL_HOLD BIT(16) +#define APCIE_PORT_INTSTAT 0x100 +#define APCIE_PORT_INTMSK 0x104 +#define APCIE_PORT_INT_LINK GENMASK(7, 4) +#define APCIE_PORT_L2 0x88 +#define APCIE_PORT_L2_ENTER 0x11 +#define APCIE_PORT_L2_BUSY BIT(0) +#define APCIE_PORT_PERST_GUARD 0x13c +#define APCIE_PORT_LINKSTS 0x208 +#define APCIE_PORT_LINKSTS_UP BIT(0) +#define APCIE_PORT_APPCLK 0x800 +#define APCIE_PORT_APPCLK_EN BIT(0) +#define APCIE_PORT_APPCLK_CGDIS BIT(8) +#define APCIE_PORT_STATUS 0x804 +#define APCIE_PORT_STATUS_READY BIT(0) +#define APCIE_PORT_PERST 0x82c +#define APCIE_PORT_PHY_LINK BIT(31) +#define APCIE_PORT_PHY_EN0 BIT(9) +#define APCIE_PORT_PHY_EN1 BIT(10) +#define APCIE_ENGINE_CTRL 0x0 +#define APCIE_ENGINE_IRQ_MASK 0x8 +#define APCIE_ENGINE_STOP BIT(0) +#define APCIE_FABRIC_STOP 0x4 +#define APCIE_PORT_STOP_US 5000 +#define APCIE_RC_ZERO 0x04 +#define APCIE_RC_READY_START 0x50 +#define APCIE_RC_READY_CFG 0x54 +#define APCIE_RC_READY_STAT 0x58 +#define APCIE_POLL_US 50000 + +/* + * Written with bits 0 and 63 by every core right before its last WFI. This is not + * CYC_OVRD, which is undefined on this SoC. + */ +#define SYS_APL_DEEP_SLEEP sys_reg(3, 5, 15, 6, 2) +#define DEEP_SLEEP_ENABLE (BIT_ULL(0) | BIT_ULL(63)) + +#define SYS_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) +#define SYS_APL_SIQ_CFG_EL1 sys_reg(3, 4, 15, 10, 4) + +/* A one-byte register write on the sleep's SPMI bus, to the PMU's USID */ +#define SPMI_CMD_WRITE (BIT(15) | 0x30) +#define SPMI_CMD_USID GENMASK(11, 8) + +/* A PMU write: the reply's vector masked, then acknowledged, so none is left raised. */ +static inline void apple_sleep_pmu_write(void __iomem *spmi, u8 usid, u16 reg, u8 val) +{ + u32 status, enable; + + enable = readl(spmi + SPMI_NOTIFY_ENABLE); + writel(enable & ~SPMI_NOTIFY_REPLY, spmi + SPMI_NOTIFY_ENABLE); + writel(((u32)reg << 16) | SPMI_CMD_WRITE | FIELD_PREP(SPMI_CMD_USID, usid), spmi + SPMI_CMD); + writel(val, spmi + SPMI_CMD); + if (!readl_poll_timeout_atomic(spmi + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 1, 1000)) + readl(spmi + SPMI_REPLY); + writel(1, spmi + SPMI_NOTIFY_ACK); + writel(enable, spmi + SPMI_NOTIFY_ENABLE); +} + +/* + * A domain is raised with the target, then AUTO_ENABLE once it reads on. Measured + * byte for byte on ANS and PCIe, for instance 0x0f002300, 0x0f00200f, 0x1f00200f. + */ +static inline int apple_sleep_ps_raise_start(void __iomem *reg) +{ + u32 now; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET, reg); + + return readl_poll_timeout_atomic(reg, now, (now & PS_ACTUAL) == PS_ACTUAL, 1, + PS_OFF_TIMEOUT_US); +} + +static inline void apple_sleep_ps_raise_finish(void __iomem *reg) +{ + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET | PS_AUTO_ENABLE, + reg); +} + +/* Read-modify-write in table order, with no write where nothing changes. */ +static inline void apple_sleep_tunable_apply_changed(void __iomem *regs, + const struct apple_tunable *t) +{ + size_t i; + + for (i = 0; i < t->sz; i++) { + void __iomem *reg = regs + t->values[i].offset; + u32 old = readl(reg); + u32 val = (old & ~t->values[i].mask) | (t->values[i].value & t->values[i].mask); + + if (val != old) + writel(val, reg); + } +} + +/* WD2 of the AON watchdog, the sleep's safety net, armed for @secs from now. */ +static inline void apple_sleep_wd2_arm(void __iomem *wdt, unsigned int secs) +{ + writel(0, wdt + WDT_WD2_CTL); + writel(0, wdt + WDT_WD2_COUNT); + writel(secs * WDT_TICKS_PER_S, wdt + WDT_WD2_ALARM); + writel(1, wdt + WDT_WD2_CTL); +} + +/* + * A SoC whose sleep doesn't fit the lists of struct apple_sleep_soc brings its own sequences. + * @probe then runs in place of the driver's own, and installs the part's suspend operations. + * @restore, if set, runs when the system comes back from a hibernation image. + * @registry_flags: how the part binds to the registry, which the driver does around @probe. + */ +struct apple_sleep_part { + int (*probe)(struct platform_device *pdev); + int (*restore)(struct device *dev); + unsigned int registry_flags; +}; + +/* sleep-t6040.c, built where the SMC driver is built in */ +#define APPLE_SLEEP_T6040 IS_BUILTIN(CONFIG_MFD_MACSMC) +extern const struct apple_sleep_part t6040_sleep_part; + +/* + * For a part with dry runs, from its .begin to its .end: 1 runs the firmware-driven device + * suspend short of what only a power cycle restores, 2 also the last steps that can still be + * taken back. apple_sleep_power_loss() and apple_sleep_last_steps() answer by it. + */ +void apple_sleep_set_dry(unsigned int level); + +/* + * The storage PCIe block's windows and tunables are on a node a loader fills. A tunable table + * @prop for the window @name, or NULL; and the window itself mapped, with its tunable if @prop + * names one, or NULL where either is missing. + */ +struct apple_tunable *apple_sleep_apcie_tunable(struct device *dev, struct device_node *np, + const char *name, const char *prop); +void __iomem *apple_sleep_apcie_block(struct device *dev, struct device_node *np, + const char *name, const char *prop, + struct apple_tunable **tunable); + +/* Each sleep begins with no answer about the wake before it. */ +void apple_sleep_wake_info_forget(void); + +/* For a debugfs file: who is registered, and how far each is. */ +void apple_sleep_registry_show(struct seq_file *s); + +/* + * The rest is for the commit point, in this order: interrupts off and one CPU up, so nothing + * takes the registry's lock. @dev is the sleep driver's, for the log. + * + * Every registered coprocessor has to be ready: returns how many aren't, each logged, and sets + * @pcie_ans_idled if ANS is among the ready ones. + */ +unsigned int apple_sleep_coprocs_refuse(struct device *dev, bool *pcie_ans_idled); +/* Each ready coprocessor's last check: the first refusal, or 0. */ +int apple_sleep_coprocs_check(struct device *dev); +bool apple_sleep_enclave_registered(void); +/* A power word that a ready coprocessor's last stop lowers: on until the commit. */ +bool apple_sleep_coproc_lowers(u32 ps); +/* The enclave's stop, last of all refusals: nonzero only if no request was sent. */ +int apple_sleep_enclave_stop(struct device *dev); +/* The last stops, those registered first with @first and then the rest. */ +void apple_sleep_coprocs_stop(bool first, bool *pcie_ans); + +#endif diff --git a/drivers/soc/apple/sleep-registry.c b/drivers/soc/apple/sleep-registry.c new file mode 100644 index 0000000000000..5903c0fbef337 --- /dev/null +++ b/drivers/soc/apple/sleep-registry.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC sleep: the coprocessor and enclave registry + * + * A driver whose coprocessor can only be stopped once a sleep is certain + * registers it here, readies it in its own suspend and leaves the last stop to + * the commit of whichever sleep controller is bound on this SoC (sleep.c, + * sleep-t6040.c). The table lives apart from the controllers so that one + * kernel can carry all of them; each binds only to its own SoC. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "sleep-internal.h" + +struct apple_sleep_coproc apple_sleep_coprocs[APPLE_SLEEP_COPROCS]; +struct apple_sleep_enclave apple_sleep_enclave; +/* Guards the table, the enclave slot and the bound controller. */ +DEFINE_MUTEX(apple_sleep_coprocs_lock); + +#define APPLE_SLEEP_REG_BOUND BIT(8) +#define APPLE_SLEEP_REG_READY BIT(9) + +/* The bound controller's flags and state. */ +static unsigned int apple_sleep_reg; + +int apple_sleep_registry_bind(unsigned int flags) +{ + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_reg & APPLE_SLEEP_REG_BOUND) + return -EBUSY; + WRITE_ONCE(apple_sleep_reg, flags | APPLE_SLEEP_REG_BOUND); + + return 0; +} + +void apple_sleep_registry_unbind(void) +{ + guard(mutex)(&apple_sleep_coprocs_lock); + WRITE_ONCE(apple_sleep_reg, 0); +} + +void apple_sleep_registry_ready(void) +{ + guard(mutex)(&apple_sleep_coprocs_lock); + WRITE_ONCE(apple_sleep_reg, apple_sleep_reg | APPLE_SLEEP_REG_READY); +} + +struct apple_sleep_coproc *apple_sleep_coproc_find(struct device *dev) +{ + int i; + + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev == dev) + return &apple_sleep_coprocs[i]; + + return NULL; +} + +/* Below EL2 a monitor owns the last mile, and this code doesn't run it. */ +int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, + int (*last_stop)(void *data), void *data) +{ + struct apple_sleep_coproc *c; + + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_coproc_find(dev)) + return -EEXIST; + c = apple_sleep_coproc_find(NULL); + if (!c) + return -ENOSPC; + c->asc = asc; + c->pcie_ans = pcie_ans; + c->last_check = NULL; + c->last_stop = last_stop; + c->data = data; + c->ready = false; + c->stopped = false; + c->first = false; + c->order = 0; + c->ps = 0; + c->dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register); + +int apple_sleep_coproc_register_first(struct device *dev, unsigned int order, + int (*last_stop)(void *data), void *data) +{ + struct apple_sleep_coproc *c; + struct device_node *np; + u32 ps = 0; + int ret; + + if (!last_stop || order >= APPLE_SLEEP_FIRST_ORDERS) + return -EINVAL; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (np) { + of_property_read_u32(np, "reg", &ps); + of_node_put(np); + } + ret = apple_sleep_coproc_register(dev, NULL, false, last_stop, data); + if (ret) + return ret; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (c) { + c->first = true; + c->order = order; + c->ps = ps; + } + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register_first); + +int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (!c) + return -ENOENT; + c->last_check = last_check; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_last_check); + +/* With no sleep controller bound, nothing would ever send the request: refuse it. */ +int apple_sleep_enclave_register(struct device *dev, int (*stop)(void *data), void *data) +{ + unsigned int reg; + + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + if (!stop) + return -EINVAL; + reg = READ_ONCE(apple_sleep_reg); + if (!(reg & APPLE_SLEEP_REG_BOUND)) + return -EOPNOTSUPP; + if (!(reg & APPLE_SLEEP_REG_ENCLAVE_ANY) && !apple_sleep_commit_stops_coprocs()) + return -EOPNOTSUPP; + + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_enclave.dev) + return -EBUSY; + apple_sleep_enclave.stop = stop; + apple_sleep_enclave.data = data; + apple_sleep_enclave.dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_register); + +void apple_sleep_enclave_unregister(struct device *dev) +{ + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_enclave.dev == dev) + memset(&apple_sleep_enclave, 0, sizeof(apple_sleep_enclave)); +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_unregister); + +void apple_sleep_coproc_unregister(struct device *dev) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (c) + memset(c, 0, sizeof(*c)); +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_unregister); + +void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (!c) + return; + c->ready = ready; + c->stopped = false; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_ready); + +bool apple_sleep_coproc_stopped(struct device *dev) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + return c && c->stopped; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_stopped); + +bool apple_sleep_commit_stops_coprocs(void) +{ + return (READ_ONCE(apple_sleep_reg) & APPLE_SLEEP_REG_READY) && is_kernel_in_hyp_mode() && + apple_sleep_power_loss(); +} +EXPORT_SYMBOL_GPL(apple_sleep_commit_stops_coprocs); + +/* + * The last thing that can fail on the way to the commit: no CPU but this one is up and + * interrupts are off, so no lock can be held. A coprocessor stopped here comes back only + * through a real sleep, so every refusal has to be behind this, and nothing after it may + * go back. In this order, on ANS itself: whatever its driver does last, + * status and control read, then 0 written twice. + */ +static void apple_sleep_coproc_run_off(struct apple_sleep_coproc *c) +{ + /* Its own last stop cleared its run bits. */ + if (!c->asc) { + c->stopped = true; + return; + } + c->status = readl(c->asc + ASC_CPU_STATUS); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + c->stopped = true; +} + +/* A power word that a ready coprocessor's last stop lowers: on until the commit. */ +bool apple_sleep_coproc_lowers(u32 ps) +{ + int i; + + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev && apple_sleep_coprocs[i].ready && + apple_sleep_coprocs[i].first && apple_sleep_coprocs[i].ps == ps) + return true; + return false; +} + +bool apple_sleep_enclave_registered(void) +{ + return apple_sleep_enclave.dev; +} + +void apple_sleep_registry_show(struct seq_file *s) +{ + int i; + + guard(mutex)(&apple_sleep_coprocs_lock); + seq_puts(s, "coprocessors registered for the commit's last stop:\n"); + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev) + continue; + seq_printf(s, " %s%s%s%s%s\n", dev_name(c->dev), c->pcie_ans ? " (ANS)" : "", + c->first ? " (first)" : "", c->last_check ? ", last check" : "", + c->ready ? ", ready" : ""); + } + seq_printf(s, "enclave sleep request left for the last refusal: %s\n", + apple_sleep_enclave.dev ? dev_name(apple_sleep_enclave.dev) : "none"); +} + +/* A registered coprocessor that isn't ready refuses the sleep. */ +unsigned int apple_sleep_coprocs_refuse(struct device *dev, bool *pcie_ans_idled) +{ + unsigned int refused = 0; + int i; + + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev) + continue; + if (c->ready) { + *pcie_ans_idled |= c->pcie_ans; + } else { + /* Nothing may lower a coprocessor's domain under its driver. */ + dev_err(dev, "refused: %s is running and its driver has not quiesced it\n", + dev_name(c->dev)); + refused++; + } + } + + return refused; +} + +/* The last refusals, each coprocessor's own: nothing has stopped yet. */ +int apple_sleep_coprocs_check(struct device *dev) +{ + int i, ret; + + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->stopped || !c->last_check) + continue; + ret = c->last_check(c->data); + if (ret) { + dev_err(dev, "refused: %s: %d\n", dev_name(c->dev), ret); + return ret; + } + } + + return 0; +} + +/* A nonzero result must mean no SLEEP was sent; post-send errors are fatal. */ +int apple_sleep_enclave_stop(struct device *dev) +{ + int ret; + + if (!apple_sleep_enclave.stop) + return 0; + ret = apple_sleep_enclave.stop(apple_sleep_enclave.data); + if (ret) + dev_err(dev, "refused before SEP SLEEP was sent: %d\n", ret); + + return ret; +} + +/* + * In two passes, after the last refusal: from the first stop only a real sleep restarts the + * GPU, so a failure here goes on to the commit. Those registered first stop whole first, + * before the off words drop under them: the GPU withdraws its PMP request there, and the + * PMP has to see that before its own stop. The rest after the match. With ANS among the ready + * ones, @pcie_ans. + */ +void apple_sleep_coprocs_stop(bool first, bool *pcie_ans) +{ + unsigned int order; + int i; + + if (first) + *pcie_ans = false; + /* Those registered first run by their order, the others in one pass over the table. */ + for (order = 0; order < (first ? APPLE_SLEEP_FIRST_ORDERS : 1); order++) { + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->first != first) + continue; + if (first && c->order != order) + continue; + *pcie_ans |= c->pcie_ans; + if (c->stopped) + continue; + c->stop_ret = c->last_stop ? c->last_stop(c->data) : 0; + apple_sleep_coproc_run_off(c); + } + } + dsb(sy); +} + +/* + * A controller that was tested with neither the lid's state at the wake nor the USB wake ports + * binds without the capability: the queries then answer as they do with no sleep driver. + */ +static bool apple_sleep_declines(unsigned int cap) +{ + unsigned int reg = READ_ONCE(apple_sleep_reg); + + return (reg & APPLE_SLEEP_REG_BOUND) && !(reg & cap); +} + +static unsigned long sleep_usb_wake_ports; + +void apple_sleep_usb_set_wake_ports(unsigned long ports) +{ + WRITE_ONCE(sleep_usb_wake_ports, ports); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_set_wake_ports); + +bool apple_sleep_usb_port_can_wake(unsigned int port) +{ + if (apple_sleep_declines(APPLE_SLEEP_REG_USB_WAKE_PORTS)) + return false; + + return port < BITS_PER_LONG && (READ_ONCE(sleep_usb_wake_ports) & BIT(port)); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_port_can_wake); + +/* Read in the SMC's noirq resume. Forgotten as each sleep begins. */ +static int sleep_lid_closed = -ENODATA; + +void apple_sleep_lid_at_wake(bool closed) +{ + WRITE_ONCE(sleep_lid_closed, closed); +} +EXPORT_SYMBOL_GPL(apple_sleep_lid_at_wake); + +int apple_sleep_lid_closed_at_wake(void) +{ + if (apple_sleep_declines(APPLE_SLEEP_REG_LID_AT_WAKE)) + return -ENODATA; + + return READ_ONCE(sleep_lid_closed); +} +EXPORT_SYMBOL_GPL(apple_sleep_lid_closed_at_wake); + +/* Set in the SMC's noirq resume. Forgotten as each sleep begins. */ +static int sleep_radio_woke = -ENODATA; + +void apple_sleep_radio_at_wake(bool woke) +{ + WRITE_ONCE(sleep_radio_woke, woke); +} +EXPORT_SYMBOL_GPL(apple_sleep_radio_at_wake); + +int apple_sleep_radio_woke(void) +{ + return READ_ONCE(sleep_radio_woke); +} +EXPORT_SYMBOL_GPL(apple_sleep_radio_woke); + +/* Each sleep begins with no answer about the wake before it. */ +void apple_sleep_wake_info_forget(void) +{ + WRITE_ONCE(sleep_lid_closed, -ENODATA); + WRITE_ONCE(sleep_radio_woke, -ENODATA); +} diff --git a/drivers/soc/apple/sleep-t6040.c b/drivers/soc/apple/sleep-t6040.c new file mode 100644 index 0000000000000..0efef072c7362 --- /dev/null +++ b/drivers/soc/apple/sleep-t6040.c @@ -0,0 +1,3168 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC system sleep: the T6040 (M4 Pro) part + * + * The T6040 sleeps through the same boot object and mailbox as the SoCs sleep.c takes from + * its lists, and nearly every step in between differs: three CPU complexes, five display + * pipes whose power states have to read off first, two storage PCIe lanes, the wake block + * and the order of the commit. So this file is a part of that driver (sleep-internal.h) with + * its own probe, state and suspend operations. The commit follows the T6040 entry as + * measured on the J616s, with what the SoC's sleep engine needs from the OS (see + * t6040_sleep_commit); the wake return mirrors the PMGR wake restore + * (see t6040_sleep_return). The registry of coprocessors and of the enclave + * (linux/soc/apple/sleep.h), the dry run level and the mailbox layout are the generic + * driver's. + * + * It refuses first: + * + * - It binds below EL2 and without the boot object's mailbox as well, and shows both as + * blockers in debugfs t6040-sleep/preflight, next to the CPUs that can park and the + * display power states the ADT lists to check before a sleep (device-check-ps-b4-sleep). + * - It offers suspend-to-RAM only to a system booted with t6040_sleep.allow_entry=1. + * Without it suspend-to-idle stays the default sleep, and it passes through here + * untouched either way. + * - Its suspend_ops .begin computes the blocker list of a suspend-to-RAM. With a blocker + * left it prints every one and returns an error before any device suspends. + * - Dry runs (t6040_sleep.dry_run, also writable at runtime): + * 1: pm_suspend_via_firmware() is set, so every driver's firmware-driven + * sleep step that can be taken back runs (apple_sleep_power_loss() is + * false, so what only a power cycle restores is left out), the CPUs + * park and answer the commit's hold, the commit point's checks run, + * and the sleep is refused there and resumed warm. + * 2: 1, plus the last steps that can still be taken back (each + * coprocessor's last check, such as ANS's check of the shutdown + * notification it sent at its suspend). The SMC's sleep commit is not + * one of them: the SMC leaves the state it puts it in only through a + * power loss of the SoC, so no dry run sends it. + * The AON watchdog's WD2 is armed from .begin to .end of a dry run + * (t6040_sleep.wd2_secs, 0 to leave it off). + * - The real commit (t6040_sleep_commit) runs only when no blocker remains, + * t6040_sleep.allow_entry=1 is given and no dry run is asked for. The AON + * watchdog's WD2 is its net (t6040_sleep.sleep_wd_secs), and over the device + * suspend and resume around it. + * + * The lid's state at the wake and the USB wake ports, which the generic driver keeps for the + * display and the port controllers, are declined: this sleep was tested without them. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "sleep-internal.h" + +/* The parameters keep the names they had when this was a driver of its own. */ +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "t6040_sleep." + +#define PS_STRIDE 8 +#define PS_MAX_PER_GROUP 16 +#define PS_SIO_FIRST 0x150 /* SOC-East: I2C0, the first SIO peripheral word */ +/* The extent of the two power word pages the driver accesses (the DT window sizes). */ +#define SOC_EAST_SIZE 0xd000 +#define SOC_PS_SIZE 0x9000 +#define PS_SIO_LAST 0x228 /* SOC-East: UART6, the last */ + +/* + * ANS's storage PCIe block: the two ST lanes (apcie_st0/apcie_st1) and their + * shared PHY (T8140 has a single lane). Taken down at the commit + * and raised on wake. The power words sit in SOC-East (t604x-pmgr.dtsi + * offsets, same bank as ps_ans); the control windows (axi2af/cio3pll/clkgen/ + * phy-block/phy-ip/rc) and the PLL/lane tunables are not in this SoC's static + * description and are published by the boot object (aura) under the apple,apcie + * node, as on the T8140. The block-internal offsets are the T8140's + * (sleep-internal.h): the same Apple PCIe IP. + */ +#define PS_ANS 0x370 /* SOC-East: ps_ans */ +#define PS_APCIE_ST0 0x108 /* SOC-East: ps_apcie_st0 */ +#define PS_APCIE_ST1 0x110 /* SOC-East: ps_apcie_st1 */ +#define PS_APCIE_SYS_ST0 0x380 /* SOC-East: ps_apcie_sys_st0 */ +#define PS_APCIE_SYS_ST1 0x388 /* SOC-East: ps_apcie_sys_st1 */ +#define PS_APCIE_PHY_SW 0x398 /* SOC-East: ps_apcie_phy_sw (the shared PHY) */ +#define PS_FAB2_SOC 0x338 /* SOC: the ATC commons' parent */ +/* The clock-gated words (TARGET 4) the commit writes, as the sleep engine asks for them. */ +#define PS_APCIE_PHY_SW_CLKGATED 0x0f000244 +#define PS_APCIE_GP_CLKGATED 0x00000244 + +/* + * cluster-ctl (ADT pmgr reg[0] + cluster-ctl-offset 0x20000 = 0x502820000), one word per + * complex, measured on the J616s: the CPU complexes E +0x0, P0 +0x4 and P1 +0x8, then the + * internal and the external display's complexes +0xc and +0x10. The low nibble 0 powers a + * complex down and 0xf powers it up; BIT4 reads set until a power-down has finished. + */ +#define CLUSTER_CTL(n) ((n) * 4) +#define CLUSTER_CTL_NR 5 /* E, P0, P1, internal and external display */ +#define CLUSTER_CTL_FIRST_WAIT 3 /* the two display complexes are polled */ +#define CLUSTER_CTL_BUSY BIT(4) +#define CLUSTER_CTL_NIBBLE GENMASK(3, 0) +#define CLUSTER_CTL_ON 0xf +#define CLUSTER_TIMEOUT_US 50000 + +/* + * SOC-PwrGate (ADT pmgr reg[0x26] = 0x50282c000): the retention bits at +0x0/+0x40 and + * the three CPM voters at +0x100/+0x108/+0x110, each BIT31. The wake restore re-asserts + * them. The entry leaves them alone: the voters already read 0x80000000 there. + */ +#define PWRGATE_ON 0x80000000 +#define PWRGATE_WAKE_0 0x000 +#define PWRGATE_WAKE_1 0x040 +#define PWRGATE_CPM_0 0x100 +#define PWRGATE_CPM_1 0x108 +#define PWRGATE_CPM_2 0x110 + +/* + * SOC-East (ADT pmgr reg[2] = 0x502280000). The SEP/retention word at +0xc00: the wake + * clears PS_DEV_DISABLE and then polls PS_BUSY clear before going on. Also holds several + * of the axi2af domains' power words. + */ +#define SOC_EAST_RETENTION 0xc00 +#define SOC_EAST_TIMEOUT_US 5000 + +/* + * The clock quiesce, whole and nothing else: two SOC-East words lose BIT28 and are + * polled until [7:4] reads their [3:0]; then the four clk-mux park words of ClkGen SOC (ADT + * pmgr reg[3] = 0x500000000, +0x38050/054/060/064), in t6040_clkgen_mux[] order, get [27:24] + * := 0 with BIT31 set, each polled until BIT30 clears. The wake writes the saved clk-mux + * words back; there is nothing else to restore. Not measured on the J616s. + */ +#define SOC_EAST_CLK_A 0x120 +#define SOC_EAST_CLK_B 0x130 +#define SOC_EAST_CLK_AUTO BIT(28) +#define CLKGEN_MUX_SET 0x80000000 +#define CLKGEN_MUX_MASK 0xf0ffffff +#define CLKGEN_MUX_WORDS 4 +#define CLKGEN_MUX_PENDING BIT(30) +#define CLK_QUIESCE_TIMEOUT_US 1000 + +/* + * The keep-alive enables: 32 words in PMGR SOC from +0x1dc000 and 32 in SOC-East from + * +0x50000, stride 0x44, BIT0 the keep-alive. Cleared right before the S2R timer, set + * again at the wake. + */ +#define T6040_KA_SOC_PA 0x5029dc000ULL +#define T6040_KA_EAST_PA 0x5022d0000ULL +#define KA_WORDS 32 +#define KA_STRIDE 0x44 +#define KA_SIZE (KA_WORDS * KA_STRIDE) +#define KA_ENABLE BIT(0) + +/* + * The SMC's AP watchdog deadline (the watchdog timer's reg[2] + 8): written 0 right after + * the AON WD2 is armed for the sleep. + */ +#define T6040_SMC_WDT_DEADLINE_PA 0x508330008ULL + +/* + * The AGX residency gate at the top of the PMGR quiesce: unless + * MiniPMGR +0x75010 has both [19:18] set (it read 0 on the J616s), the GPU's AFR power + * word (pmgr_gfx 0x288e80000 + 0x100, ADT power-device AFR) is lowered when its + * target is not 0: [19:16] := 4 and polled, [7:4] polled to 4, then the word := 0 + * and [7:4] polled to 0. + */ +#define T6040_AGX_GATE_PA 0x508375010ULL +#define AGX_GATE_RESIDENT GENMASK(19, 18) +#define T6040_AFR_PS_PA 0x288e80100ULL +#define AGX_TIMEOUT_US 10000 + +/* + * Before the SoC-specific steps of the PMGR quiesce, in order: + * - the ACC quiesce of c = E, P0, P1: APSC off, ACC +0xe20020 BIT23 := 1 (64-bit), + * poll BIT7 clear (CPM gating after it reads back already set: not written); + * - features off: cpu-fixed-freq-pll-relock, BIT42 := 0 in the E and P0 + * ACC words; then the TVMs disp2, afr, dcs (two), gfx (two), disp, soc: control + * BIT0 := 0, status polled until BIT1 clears. + * The PMC's DCS floor vote is not touched: an access to that word hangs the bus. Every poll + * here is bounded. + */ +static const u64 t6040_acc_apsc_pa[] = { 0x210e20020ULL, 0x211e20020ULL, 0x212e20020ULL }; +#define ACC_APSC_DISABLE BIT_ULL(23) +#define ACC_APSC_BUSY BIT_ULL(7) +#define ACC_PLL_RELOCK BIT_ULL(42) +#define ACC_PLL_RELOCK_COMPLEXES 2 +#define QUIESCE_TIMEOUT_US 2000 + +static const struct { + u64 ctl; + u64 status; +} t6040_tvm[] = { + { 0x502920000ULL, 0x502920250ULL }, /* disp2-tvm */ + { 0x288e95000ULL, 0x288e953e0ULL }, /* afr-tvm */ + { 0x502908000ULL, 0x502908250ULL }, /* dcs-tvm */ + { 0x502910000ULL, 0x502910250ULL }, + { 0x288e94000ULL, 0x288e943e0ULL }, /* gfx-tvm */ + { 0x288e94400ULL, 0x288e947e0ULL }, + { 0x502918000ULL, 0x502918250ULL }, /* disp-tvm */ + { 0x502900000ULL, 0x502900250ULL }, /* soc-tvm */ +}; + +#define TVM_ENABLE BIT(0) +#define TVM_BUSY BIT(1) + +/* + * The PMC's bandwidth-request and boost interface enables of the display pipes + * (0x5048ac020 and +0x8), one bit per pipe. + */ +#define T6040_PMC_BWR_PA 0x5048ac020ULL +#define PMC_BOOST 0x8 + +/* + * The PMU over SPMI (spmi = 0x509014000, spmi-a0 = 0x509188000), written as on the T8140. + * Measured on the J616s: the boot stage is set to 0x50 by an extended write to + * PMU_BOOT_STAGE, and NOTIFY_ENABLE is 0x2000001 on both controllers' +0x40. + */ +/* The PMU's USID on the sleep's SPMI bus, and the boot stage that announces a sleep */ +#define PMU_USID 0xe +#define PMU_BOOT_STAGE 0xf801 /* T6040 slave/reg */ +#define PMU_BOOT_STAGE_SLEEP 0x50 +/* The response vector's enable for the sleep */ +#define SPMI_NOTIFY_ENABLE_SLEEP 0x2000001 + +/* + * MasterMini, the block of the always-on PMGR (ADT pmgr reg[0x37] = 0x508900000). + * The S2R timer at +0x4000 (physical 0x508904000) is the commit: writing 1 arms + * the SoC sleep engine. Measured on the J616s: exactly this write. + */ +#define MASTERMINI_S2R_TIMER 0x4000 + +/* + * AOP-global (the AP controls, window 0x508000000). Mapped from the + * S2R-control word 0x508004000: +0x0 the S2R control (cleared to 0 on wake), + * +0xc..+0x2c the 9 wake-source masks (armed before + * sleep), +0x30..+0x50 the 9 wake-source status words (read on wake). + */ +#define AOP_S2R_CTRL 0x00 +#define AOP_WAKE_MASK(w) (0x0c + (w) * 4) +#define AOP_WAKE_STATUS(w) (0x30 + (w) * 4) +#define AOP_WAKE_WORDS 9 + +/* axi2af fabric-bridge retention re-arm. */ +#define AXI2AF_BRIDGE_104 0x104 +#define AXI2AF_BRIDGE_108 0x108 + +/* The internal display: its PMGR words and its DART. */ +#define T6040_DCP_DART_PA 0x402340000ULL +#define T6040_DISP_PWRGATE_PA 0x5022a1000ULL +#define DISP_PWRGATE_A 0x1c /* := 3 before DISP_SYS goes down, 0 after it is up */ +#define DISP_PWRGATE_B 0x60 /* := 0 before DISP_SYS goes down, 1 after it is up */ +#define PS_DISP_FE 0x368 /* soc_east */ +#define PS_DISP_SYS 0x258 /* soc_east */ +#define DISP_PS_TIMEOUT_US 8000 +#define DISP_CPU_GATE_US 50000 + +/* The GP PCIe (t602x port layout, as T8140's radio port): power words in soc_east. */ +#define PS_APCIE_GP 0x100 +#define PS_APCIE_SYS_GP 0x390 +#define GP_PIN_DATA BIT(0) +/* The PMC's DCP voters: the count of DCPs powered (+0) and a second voter (+4), both 0 at sleep */ +#define T6040_PMC_DCP_PA 0x5048a9c14ULL + +/* The axi2af fabric-bridge retention domains, t6040_axi2af[] */ +enum { + AXI2AF_ATC0_COMMON, + AXI2AF_ATC1_COMMON, + AXI2AF_ATC2_COMMON, + AXI2AF_ATC3_COMMON, + AXI2AF_DISP_SYS, + AXI2AF_DISPEXT0_SYS, + AXI2AF_DISPEXT1_SYS, + AXI2AF_DISPEXT2_SYS, + AXI2AF_DISPEXT3_SYS, + AXI2AF_ANS, + AXI2AF_AVD_SYS, + AXI2AF_VENC0_SYS, + AXI2AF_VENC1_SYS, + AXI2AF_JPG, + AXI2AF_APCIE_SYS_GP, + T6040_AXI2AF_DOMAINS +}; + +/* + * The AON watchdog's WD2, the sleep safety net: ADT wdt reg[0] = 0x50836c000, with the + * T8140's registers. Linux's own watchdog driver runs WD1 (+0x10..+0x1c), not this one. + * ALARM is 32 bits at 24 MHz. + */ +#define WDT_MAX_SECS 178U +#define T6040_WDT_PA 0x50836c000ULL /* the address t6040_resume_stub writes */ + +struct t6040_sleep_group { + const char *name; + struct regmap *regmap; + u32 first; /* the pipe's CPU */ + u32 last; /* its last sub-block */ +}; + +/* A device the commit point looks at: bound or not, and its own power word. */ +struct t6040_sleep_watch { + const char *compatible; + struct regmap *ps_regmap; + u32 ps_offset; + bool has_ps; + bool bound; +}; + +enum { + T6040_WATCH_SEP, + T6040_WATCH_ANS, + T6040_WATCH_COUNT, +}; + +/* + * ANS's storage-apcie control windows and PLL/lane tunables, mapped from the + * apple,apcie node the boot object publishes (t6040_sleep_map_apcie). Every + * pointer is NULL until a loader fills the node: @valid is false then, and a + * real sleep with ANS stopped refuses. The two ST lanes share this one PHY. + */ +struct t6040_apcie_st { + void __iomem *axi; /* reg-name "axi2af" (shared) */ + void __iomem *cio3pll; /* reg-name "cio3pll" (shared) */ + void __iomem *clkgen; /* reg-name "clkgen" (shared) */ + void __iomem *phy; /* reg-name "phy-block" (shared PHY) */ + void __iomem *phy_ip; /* reg-name "phy-ip" */ + void __iomem *rc; /* reg-name "rc" */ + struct apple_tunable *axi_tunable; /* apple,tunable-axi2af */ + struct apple_tunable *cio3pll_tunable; /* apple,tunable-cio3pll */ + struct apple_tunable *clkgen_tunable; /* apple,tunable-clkgen */ + struct apple_tunable *phy_tunable; /* apple,tunable-phy */ + struct apple_tunable *phy_ip_pll; /* apple,tunable-phy-ip-pll */ + struct apple_tunable *phy_ip_lanes; /* apple,tunable-phy-ip-auspma */ + bool valid; +}; + +struct t6040_sleep { + struct device *dev; + struct t6040_sleep_group *groups; + unsigned int ngroups; + /* The dry run level of the suspend in progress, from .begin to .end; 0 otherwise. */ + unsigned int dry; + void __iomem *cpu_start; + void __iomem *wdt; + /* The DT's wdt block is the one t6040_resume_stub hardcodes. */ + bool wdt_stub_ok; + /* The commit's and the wake's SoC blocks (see the #defines). Optional. */ + void __iomem *cluster_ctl; + void __iomem *pwrgate; + void __iomem *dcp_dart; /* dart-dcp, for its transaction idle wait */ + void __iomem *disp_pwrgate; /* internal display retention window 0x5022a1000 */ + bool disp_down; /* t6040_disp_off() lowered the internal display */ + void __iomem *pmc_dcp; /* PMS pmc-dcp voter pair 0x5048a9c14/c18 */ + u32 pmc_dcp_saved[2]; + u32 pmc_if_saved[2]; /* BWR and boost interface enables at the commit */ + bool pmc_if_valid; + /* The GP PCIe host's two ports (Wi-Fi/BT, SD) for the commit's port stop (T8140's). */ + void __iomem *gp_port[2]; + void __iomem *gp_phy[2]; + void __iomem *gp_pins; + int gp_perst[2]; + bool gp_perst_low[2]; + void __iomem *clkgen; + void __iomem *spmi0; + void __iomem *spmi_a0; + void __iomem *mastermini; + void __iomem *dcs[16]; /* DCS(n,k): 0x220200000 + n * 0x2000000 + k * 0x200000 */ + void __iomem *atc_cfg[4]; /* ATCPHY_CFG(n) 0x383200000 + n * 0x8000000 */ + void __iomem *aop_ps; /* MiniPMGR AOP + audio island PS words 0x508304000..0x158 */ + u32 sio_ps_saved[(PS_SIO_LAST - PS_SIO_FIRST) / PS_STRIDE + 1]; + bool sio_ps_saved_valid; + /* The SOC-East and SOC power word pages: fabrics, ATC commons, DCS, SIO. */ + void __iomem *soc_east; /* 0x502280000 */ + void __iomem *soc_ps; /* 0x502800000 */ + void __iomem *aop_global; + void __iomem *axi2af_bridge[T6040_AXI2AF_DOMAINS]; + struct aura_s2r_mailbox *mbox; + u64 mbox_pa; + bool power_ops; + bool wd2_armed; + /* Set by .begin for a suspend to RAM; .end only undoes what .begin did. */ + bool mem_begun; + u64 siq_cfg; + u64 cntkctl; + /* Saved across the commit, restored on wake. */ + u32 clkgen_mux_saved[CLKGEN_MUX_WORDS]; + bool clkgen_saved; + void __iomem *ka_soc; + void __iomem *ka_east; + u64 ka_on_before; /* keep-alive bits set before the commit cleared them */ + void __iomem *smc_wdt_deadline; + void __iomem *agx_gate; + void __iomem *s2r_lo_count; + void __iomem *acc_apsc[3]; + void __iomem *tvm_ctl[8]; + void __iomem *tvm_status[8]; + void __iomem *pmc_bwr; + void __iomem *afr_ps; + u32 cluster_saved[CLUSTER_CTL_NR]; + u32 wake_mask_saved[AOP_WAKE_WORDS]; + bool wake_mask_saved_valid; + u32 wake_status[AOP_WAKE_WORDS]; + bool returned; + /* ANS's storage ST lanes: windows + whether a commit stopped ANS. */ + struct t6040_apcie_st apcie_st; + bool pcie_ans_stopped; + int pcie_ans_on_ret; + u32 apcie_wake_step; + u32 gp_phy_saved[2], gp_appclk_saved[2], gp_perst_saved[2]; + bool gp_saved[2]; + unsigned int gp_clk_timeouts; + unsigned int aop_ps_timeouts; + u32 aop_ps_saved[44]; + bool aop_ps_saved_valid; + struct t6040_sleep_watch watch[T6040_WATCH_COUNT]; +}; + +/* + * 0: no dry run. 1: the firmware-driven device suspend short of what only a power + * cycle restores, parking and the commit's hold, refused at the commit point. 2: 1 and + * the last steps that can still be taken back. Read at each suspend's .begin. + */ +static unsigned int dry_run; +module_param(dry_run, uint, 0644); +MODULE_PARM_DESC(dry_run, + "Dry run level: 1 firmware-driven device sleep refused at the commit point, 2 also the last steps"); + +static bool allow_entry; +module_param(allow_entry, bool, 0444); +MODULE_PARM_DESC(allow_entry, + "Reach the real commit if no blocker remains"); + +/* + * The AON WD2 net armed at the commit. 5 s covers the reconfig; a longer + * net tells "the SoC never slept" (reset at the net) from "it slept and the wake + * failed" (reset at the wake source). Never off: 1 to 178. + */ +static unsigned int sleep_wd_secs = 30; +module_param(sleep_wd_secs, uint, 0644); +MODULE_PARM_DESC(sleep_wd_secs, + "AON WD2 net armed at the S2R commit, in seconds (default 30, clamped to 1..178)"); + +static unsigned int wd2_secs = WDT_SUSPEND_SECS; +module_param(wd2_secs, uint, 0644); +MODULE_PARM_DESC(wd2_secs, + "Arm the AON WD2 for this many seconds from .begin to .end of a dry run (0: off, at most 178)"); + +static struct t6040_sleep *t6040_sleep; + +static void t6040_iounmap(void *p) +{ + iounmap((void __iomem __force *)p); +} + +/* + * Every window here non-posted (nGnRnE), as the PMGR, PMC and coprocessor blocks + * require. No region request: these overlap other drivers. + */ +static void __iomem *t6040_ioremap_np(struct device *dev, phys_addr_t pa, size_t size) +{ + void __iomem *p = ioremap_np(pa, size); + + if (!p || devm_add_action_or_reset(dev, t6040_iounmap, (void __force *)p)) + return NULL; + return p; +} + +/* The generic driver answers apple_sleep_power_loss() by it. */ +static void t6040_sleep_set_dry(struct t6040_sleep *s, unsigned int dry) +{ + s->dry = dry; + apple_sleep_set_dry(dry); +} + +/* + * In the power-state check, a sub-block is on while its TARGET is not 0. A + * pipe's CPU word never counts: it stays under automatic power gating (taking it out + * of automatic mode raises an SError) and the SoC power-down drops it. + */ +static bool t6040_sleep_ps_on(u32 val, bool cpu) +{ + return !cpu && FIELD_GET(PS_TARGET, val); +} + +static unsigned int t6040_sleep_show_group(struct seq_file *m, const struct t6040_sleep_group *g) +{ + unsigned int on = 0; + u32 off; + + for (off = g->first; off <= g->last; off += PS_STRIDE) { + bool cpu = off == g->first; + u32 val; + int ret; + + ret = regmap_read(g->regmap, off, &val); + if (ret) { + /* Unreadable counts as on: the refusal fails closed. */ + if (m) + seq_printf(m, " %-12s +%#05x read failed (%d)\n", g->name, off, + ret); + on++; + continue; + } + if (t6040_sleep_ps_on(val, cpu)) + on++; + if (m) + seq_printf(m, " %-12s +%#05x %s %#010x target %lu actual %lu%s\n", g->name, + off, cpu ? "cpu " : "child", val, FIELD_GET(PS_TARGET, val), + FIELD_GET(PS_ACTUAL, val), + t6040_sleep_ps_on(val, cpu) ? " ON" : ""); + } + + return on; +} + +/* How many secondaries can leave the kernel (park), and the boot ops' name. */ +static unsigned int t6040_sleep_parkable(const char **name) +{ + unsigned int cpu, parkable = 0; + + for_each_possible_cpu(cpu) { + const struct cpu_operations *ops = get_cpu_ops(cpu); + + if (cpu && ops && ops->cpu_die && ops->cpu_disable) + parkable++; + } + if (name) + *name = get_cpu_ops(0) ? get_cpu_ops(0)->name : "?"; + + return parkable; +} + +/* + * The live blockers, recomputed each time: not at EL2, no boot-object return + * path, a CPU that cannot park, or a display pipe still on. Returns the count, + * and lists each when @s is given. @disp_out, if not NULL, gets the subtotal of + * display-pipe blockers -- these clear during device suspend (apple-dcp's + * .suspend stops the DCP and lowers its domains), so .begin excludes them and + * only .enter, after the device suspend, enforces them. + */ +static unsigned int t6040_sleep_live_blockers(struct t6040_sleep *s, struct seq_file *m, + unsigned int *disp_out) +{ + /* Mapped and aura's: after a resume from disk, this boot's (t6040_sleep_restore()). */ + bool mailbox = s->mbox; + bool el2 = is_kernel_in_hyp_mode(); + unsigned int i, blockers = 0, disp = 0, parkable; + const char *ops_name; + + parkable = t6040_sleep_parkable(&ops_name); + + if (m) { + seq_printf(m, "kernel at EL2: %s\n", el2 ? "yes" : "no"); + seq_printf(m, "boot object return path (/chosen/aurora,s2r-mailbox): %s\n", + mailbox ? "yes" : "no"); + seq_printf(m, "secondaries that can leave the kernel: %u of %u (%s)\n", parkable, + num_possible_cpus() - 1, ops_name); + } + blockers += !el2 + !mailbox + (parkable != num_possible_cpus() - 1); + + if (m) + seq_puts(m, "power states checked before sleep (device-check-ps-b4-sleep):\n"); + for (i = 0; i < s->ngroups; i++) { + unsigned int on = t6040_sleep_show_group(m, &s->groups[i]); + + if (m) + seq_printf(m, " %-12s %u on\n", s->groups[i].name, on); + disp += !!on; + } + + if (disp_out) + *disp_out = disp; + return blockers + disp; +} + +/* The display groups' on-state, one line per group, for the log at the commit point. */ +static unsigned int t6040_sleep_log_groups(struct t6040_sleep *s) +{ + unsigned int i, blockers = 0; + + for (i = 0; i < s->ngroups; i++) { + const struct t6040_sleep_group *g = &s->groups[i]; + u32 off, on_mask = 0, val; + unsigned int bit = 0; + + for (off = g->first; off <= g->last; off += PS_STRIDE, bit++) { + if (regmap_read(g->regmap, off, &val) || + t6040_sleep_ps_on(val, off == g->first)) { + on_mask |= BIT(bit); + dev_info(s->dev, " %s +%#x = %#x\n", g->name, off, val); + } + } + if (on_mask) + dev_info(s->dev, "%-8s ON (on mask %#x, cpu first)\n", g->name, + on_mask); + blockers += !!on_mask; + } + + return blockers; +} + +/* Log the blockers outside the display, for a refused or dry .begin. */ +static void t6040_sleep_log_blockers(struct t6040_sleep *s) +{ + const char *ops_name; + + dev_info(s->dev, "EL2 %s, s2r-mailbox %s, parkable %u/%u (%s)\n", + is_kernel_in_hyp_mode() ? "yes" : "no", s->mbox ? "yes" : "no", + t6040_sleep_parkable(&ops_name), num_possible_cpus() - 1, ops_name); +} + +/* + * CPU parking power ops. MPIDR Aff1 is the cluster (E=0, P0=1, + * P1=2), Aff0 the core. The start and stop bit is the core's bit in the ADT + * function-enable_core masks, which is its cpu-id: the E cluster has four + * slots (bits 0-3), each P cluster six (P0 bits 4-9, P1 bits 10-15; one slot + * of each is harvested on J616s). Four-core banks (4 * cluster + core) would + * give P1 core 0 the bit of P0 core 4. + */ +static const u8 t6040_cluster_first_bit[] = { 0, 4, 10 }; +static const u8 t6040_cluster_slots[] = { 4, 6, 6 }; + +static unsigned int t6040_cpu_cluster(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 1); +} + +static unsigned int t6040_cpu_core(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 0); +} + +static u32 t6040_cpu_start_bit(u64 mpidr) +{ + unsigned int cluster = t6040_cpu_cluster(mpidr), core = t6040_cpu_core(mpidr); + + if (WARN_ON_ONCE(cluster >= ARRAY_SIZE(t6040_cluster_first_bit) || + core >= t6040_cluster_slots[cluster])) + return 0; + return BIT(t6040_cluster_first_bit[cluster] + core); +} + +/* + * Each core's CoreSight external debug block (ADT cpuN coresight-reg: 0x210010000 + + * cluster * 0x1000000 + core * 0x100000). Every CPU, boot and secondary, + * unlocks it (EDLAR) and writes EDPRCR := 0 right before its deep WFI, so that + * the debug logic does not prevent power gating. With + * EDPRCR.CORENPDRQ left set, a core only emulates its power-down. + */ +#define T6040_ED_BASE_PA 0x210010000ULL +#define T6040_ED_CLUSTER_STRIDE 0x1000000ULL +#define T6040_ED_CORE_STRIDE 0x100000ULL +#define ED_EDPRCR 0x310 +#define ED_EDLAR 0xfb0 +#define ED_LOCK_ACCESS_KEY 0xc5acce55 +#define T6040_ED_CLUSTERS ARRAY_SIZE(t6040_cluster_slots) +#define T6040_ED_CORES 6 /* the most slots of a cluster */ + +static void __iomem *t6040_ed[T6040_ED_CLUSTERS][T6040_ED_CORES]; + +static void t6040_ed_map(struct device *dev) +{ + unsigned int cpu, n = 0; + + for_each_possible_cpu(cpu) { + u64 mpidr = cpu_logical_map(cpu); + unsigned int c = t6040_cpu_cluster(mpidr), k = t6040_cpu_core(mpidr); + + if (c >= T6040_ED_CLUSTERS || k >= T6040_ED_CORES || t6040_ed[c][k]) + continue; + t6040_ed[c][k] = t6040_ioremap_np(dev, T6040_ED_BASE_PA + + c * T6040_ED_CLUSTER_STRIDE + + k * T6040_ED_CORE_STRIDE, 0x1000); + n += !!t6040_ed[c][k]; + } + dev_dbg(dev, "external debug windows mapped for %u cores\n", n); +} + +static void t6040_ed_release(void) +{ + u64 mpidr = read_cpuid_mpidr(); + unsigned int c = t6040_cpu_cluster(mpidr), k = t6040_cpu_core(mpidr); + void __iomem *ed; + + if (c >= T6040_ED_CLUSTERS || k >= T6040_ED_CORES) + return; + ed = t6040_ed[c][k]; + if (!ed) + return; + writel_relaxed(ED_LOCK_ACCESS_KEY, ed + ED_EDLAR); + writel_relaxed(0, ed + ED_EDPRCR); +} + +static void t6040_deep_wfi_arm(void) +{ + /* A timer left running ends the WFI and costs power until reset. */ + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntp_ctl_el0); + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntv_ctl_el0); + t6040_ed_release(); + sysreg_clear_set_s(SYS_APL_DEEP_SLEEP, 0, DEEP_SLEEP_ENABLE); +} + +/* A core's last instructions: nothing brings it back but a reset. */ +static void __noreturn t6040_deep_wfi(void) +{ + t6040_deep_wfi_arm(); + + while (true) { + dsb(sy); + isb(); + wfi(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +static void t6040_cpu_stop(u64 mpidr) +{ + u32 bit = t6040_cpu_start_bit(mpidr); + + if (t6040_sleep && t6040_sleep->cpu_start && bit) + writel(bit, t6040_sleep->cpu_start + CPU_START_STOP); + dsb(sy); +} + +static void t6040_cpu_off(u64 mpidr) +{ + t6040_deep_wfi(); +} + +static int t6040_cpu_on(u64 mpidr) +{ + struct t6040_sleep *s = t6040_sleep; + unsigned int cluster = t6040_cpu_cluster(mpidr); + u32 bit = t6040_cpu_start_bit(mpidr); + + if (!s) + return -ENODEV; + if (!bit) + return -EINVAL; + writel(bit, s->cpu_start + CPU_START_SYS); + writel(BIT(t6040_cpu_core(mpidr)), s->cpu_start + CPU_START_CLUSTER(cluster)); + + return 0; +} + +static const struct spin_table_power_ops t6040_cpu_power_ops = { + .cpu_stop = t6040_cpu_stop, + .cpu_off = t6040_cpu_off, + .cpu_on = t6040_cpu_on, +}; + +/* + * The commit's hold needs them. Registered, they also let kexec take parking + * CPUs as able to leave the kernel, for the rest of this boot. Nothing calls + * @cpu_stop or @cpu_off but a commit, and @cpu_on starts only a CPU a commit + * stopped. + */ +static void t6040_sleep_power_ops(struct t6040_sleep *s) +{ + if (s->power_ops) + return; + if (!spin_table_set_power_ops(&t6040_cpu_power_ops)) { + s->power_ops = true; + dev_dbg(s->dev, "CPU parking power ops registered\n"); + } +} + +static void t6040_sleep_set_wd2(struct t6040_sleep *s, unsigned int secs) +{ + apple_sleep_wd2_arm(s->wdt, secs); + s->wd2_armed = true; +} + +static void t6040_sleep_clear_wd2(struct t6040_sleep *s) +{ + if (!s->wd2_armed) + return; + writel(0, s->wdt + WDT_WD2_CTL); + s->wd2_armed = false; +} + +/* + * In process context, after the device suspend: which watched devices are + * bound. The commit point runs with interrupts off and only reads power words. + */ +static void t6040_sleep_watch_bound(struct t6040_sleep *s) +{ + int i; + + for (i = 0; i < T6040_WATCH_COUNT; i++) { + struct t6040_sleep_watch *w = &s->watch[i]; + struct device_node *np = of_find_compatible_node(NULL, NULL, w->compatible); + struct platform_device *pdev = np ? of_find_device_by_node(np) : NULL; + + w->bound = pdev && device_is_bound(&pdev->dev); + if (pdev) + put_device(&pdev->dev); + of_node_put(np); + } +} + +/* Its power word's actual state is on. Unknown counts as on. */ +static bool t6040_sleep_watch_on(struct t6040_sleep_watch *w, u32 *val) +{ + *val = 0; + if (!w->has_ps || regmap_read(w->ps_regmap, w->ps_offset, val)) + return true; + return FIELD_GET(PS_ACTUAL, *val); +} + +/* + * The checks a real commit makes, read-only: every registered coprocessor + * quiesced by its driver, one CPU up and it the boot object's, the SEP bound + * with its sleep request left here, ANS stopped only by its driver, and the + * display pipes off. Logs each failure and returns the count. + */ +static unsigned int t6040_sleep_refuse(struct t6040_sleep *s) +{ + struct t6040_sleep_watch *sep = &s->watch[T6040_WATCH_SEP]; + struct t6040_sleep_watch *ans = &s->watch[T6040_WATCH_ANS]; + bool ans_idled = false; + unsigned int refused; + u32 val; + + refused = apple_sleep_coprocs_refuse(s->dev, &ans_idled); + + if (num_online_cpus() != 1 || !s->mbox || + (read_cpuid_mpidr() & MPIDR_HWID_BITMASK) != s->mbox->boot_mpidr) { + dev_err(s->dev, "refused: wrong CPU, other CPUs online, or no mailbox\n"); + refused++; + } + + if (!sep->bound) { + /* Nobody told it of the sleep: a commit with it running never returns. */ + t6040_sleep_watch_on(sep, &val); + dev_err(s->dev, "refused: sep isn't bound (state %#x)\n", val); + refused++; + } else if (!apple_sleep_enclave_registered()) { + dev_err(s->dev, "refused: sep's sleep request isn't left for the last refusal\n"); + refused++; + } + + /* Only ANS's driver can stop it, and it doesn't come back without a real sleep. */ + if (!ans_idled && t6040_sleep_watch_on(ans, &val)) { + dev_err(s->dev, "refused: ANS is on (%#x) with no driver to stop it\n", val); + refused++; + } + + /* + * ANS would be stopped, but its ST lanes have no way back without the + * apple,apcie control windows + PLL/lane tunables: refused. + */ + if (ans_idled && !s->apcie_st.valid) { + dev_err(s->dev, "refused: ANS would stop but the storage-apcie windows/tunables are absent (apple,apcie)\n"); + refused++; + } + + /* The engine waits for the display CPUs with no timeout. */ + if (t6040_sleep_log_groups(s)) + refused++; + + return refused; +} + +/* + * The AOP and audio island's power words (MiniPMGR 0x508304000 + 0x000..0x158). The island + * is off after a sleep, and the AOP's resume, which reads its ASC first, faults on it. The + * words are read in .begin and put back at the wake, parents first: a word that was on goes + * to a manual target 0xf (in automatic mode it gates again before the AOP's driver gets to + * its ASC), any other gets its saved value if that asks for power. The table + * is the power-down order, children first. + */ +#define T6040_AOP_PS_PA 0x508304000ULL +static const u16 t6040_aop_ps_order[] = { + 0x0b0, 0x128, 0x120, 0x158, 0x150, 0x148, 0x140, 0x138, 0x130, 0x118, 0x0c0, 0x0c8, + 0x0e8, 0x0f0, 0x0f8, 0x100, 0x108, 0x110, 0x118, 0x0e0, 0x0d8, 0x0d0, 0x0b8, 0x090, + 0x020, 0x098, 0x0a0, 0x0a8, 0x028, 0x030, 0x068, 0x070, 0x078, 0x080, 0x038, 0x040, + 0x048, 0x058, 0x060, 0x088, 0x050, 0x010, 0x018, 0x008, 0x000, +}; + +#define T6040_AOP_PS_WORDS 44 + +static void t6040_aop_ps_save(struct t6040_sleep *s) +{ + int i; + + s->aop_ps_saved_valid = false; + if (!s->aop_ps) + return; + for (i = 0; i < T6040_AOP_PS_WORDS; i++) + s->aop_ps_saved[i] = readl(s->aop_ps + PS_STRIDE * i); + s->aop_ps_saved_valid = true; +} + +static void t6040_aop_ps_restore(struct t6040_sleep *s) +{ + int k; + + s->aop_ps_timeouts = 0; + if (!s->aop_ps || !s->aop_ps_saved_valid) + return; + for (k = ARRAY_SIZE(t6040_aop_ps_order) - 1; k >= 0; k--) { + u16 off = t6040_aop_ps_order[k]; + void __iomem *r = s->aop_ps + off; + u32 v, want = s->aop_ps_saved[off / 8], now = readl(r); + + if (FIELD_GET(PS_ACTUAL, want)) { + if (FIELD_GET(PS_ACTUAL, now) == PS_ACTIVE && !(now & PS_AUTO_ENABLE)) + continue; + writel((now & ~PS_AUTO_ENABLE) | PS_TARGET, r); + } else { + if (FIELD_GET(PS_ACTUAL, now) || !FIELD_GET(PS_TARGET, want)) + continue; + writel(want, r); + } + if (readl_poll_timeout_atomic(r, v, FIELD_GET(PS_ACTUAL, v), 1, 2000)) + s->aop_ps_timeouts++; + } +} + +/* + * The power loss resets every power word, and only the domains a Linux driver owns are + * raised again when their devices resume. Here the SIO peripheral words (I2C0 .. UART6 in + * SOC-East) that were fully on at the commit + * and came back lower are raised again, parents first. On J616s those are what the boot + * loader left on and no driver claims: I2C0, SPI0/1/3/4/5 (SPI4 goes to the internal panel), + * the UARTs, AES. Nothing else: every other word is its owner's to raise. + */ +static void t6040_sio_ps_save(struct t6040_sleep *s) +{ + unsigned int i; + + s->sio_ps_saved_valid = false; + for (i = 0; i < ARRAY_SIZE(s->sio_ps_saved); i++) + s->sio_ps_saved[i] = readl(s->soc_east + PS_SIO_FIRST + i * PS_STRIDE); + s->sio_ps_saved_valid = true; +} + +static void t6040_sio_ps_restore(struct t6040_sleep *s) +{ + unsigned int i, n, raised = 0, failed = 0; + u32 val, cur = 0; + + if (!s->sio_ps_saved_valid) + return; + s->sio_ps_saved_valid = false; + for (i = 0; i < ARRAY_SIZE(s->sio_ps_saved); i++) { + u32 off = PS_SIO_FIRST + i * PS_STRIDE; + void __iomem *w = s->soc_east + off; + u32 saved = s->sio_ps_saved[i]; + + if (FIELD_GET(PS_TARGET, saved) != PS_ACTIVE || + FIELD_GET(PS_ACTUAL, saved) != PS_ACTIVE) + continue; + val = readl(w); + if (FIELD_GET(PS_TARGET, val) == PS_ACTIVE) + continue; + writel(val | PS_TARGET, w); + /* Timekeeping is still suspended here: count the wait, no ktime. */ + for (n = 0; n < 1000; n++) { + cur = readl(w); + if (FIELD_GET(PS_ACTUAL, cur) == PS_ACTIVE) + break; + udelay(10); + } + if (n == 1000) + failed++; + raised++; + dev_dbg(s->dev, "wake: SOC-East power word +%#x: %#x at the commit, %#x after the wake, now %#x\n", + off, saved, val, cur); + } + if (failed) + dev_warn(s->dev, "wake: %u of %u SIO power word(s) raised again did not come on\n", + failed, raised); +} + +/* + * The 4 clk-mux park words (ClkGen SOC + these offsets), from + * the clock quiesce; the offsets skip 0x58/0x5c. + */ +static const u32 t6040_clkgen_mux[CLKGEN_MUX_WORDS] = { + 0x38060, 0x38064, 0x38054, 0x38050, +}; + +/* + * The 9 AOP-global wake-source masks armed before an S2R + * (the values are decoded from the ADT's ap-wake-sources). + * valid & ~enabled per word. + */ +static const u32 t6040_wake_mask[AOP_WAKE_WORDS] = { + 0xe5ff5fff, 0xff42edff, 0xffffffff, 0x3ffffbdf, 0xf7ffffc0, + 0xffffff7f, 0xffffffff, 0xffffffff, 0x00003fff, +}; + +/* + * The hardware S2R-lo entry counter: bits 31:3 of MiniPMGR + 0x70970. It tells "the SoC + * entered S2R" from "the engine never finished". Read-only, with the AOP-global S2R + * control, wake masks and wake status, at debug level: at probe, for the boot after a + * failed sleep, and in a dry run. Mapped once at probe: the dry run reads it with + * interrupts off. + */ +#define T6040_S2R_LO_COUNT_PA 0x508370970ULL + +static void t6040_sleep_log_s2r_state(struct t6040_sleep *s, const char *when) +{ + u32 raw = s->s2r_lo_count ? readl(s->s2r_lo_count) : 0; + u32 m[AOP_WAKE_WORDS] = {}, st[AOP_WAKE_WORDS] = {}, ctl = 0; + int i; + + ctl = readl(s->aop_global + AOP_S2R_CTRL); + for (i = 0; i < AOP_WAKE_WORDS; i++) { + m[i] = readl(s->aop_global + AOP_WAKE_MASK(i)); + st[i] = readl(s->aop_global + AOP_WAKE_STATUS(i)); + } + dev_dbg(s->dev, "S2R state at %s: S2R-lo count %u (raw %#x), AOP-global ctl %#x\n", + when, raw >> 3, raw, ctl); + dev_dbg(s->dev, "S2R state at %s: wake masks %*ph\n", when, + (int)sizeof(m), m); + dev_dbg(s->dev, "S2R state at %s: wake status %*ph\n", when, + (int)sizeof(st), st); +} + +/* + * The 15 domains with an axi2af fabric bridge to re-arm. Each domain's power word is in + * SOC (soc_ps, base 0x502800000) or SOC-East (soc_east, base 0x502280000); its + * fabric bridge is a separate window. On wake every domain that is powered up + * (PS TARGET == 0xf) gets bridge +0x104/+0x108 bit0 set, or the bridge drops + * its traffic. + */ +static const struct t6040_axi2af_domain { + const char *name; + bool east; /* power word in soc_east (true) or soc_ps (false) */ + u32 ps_off; + phys_addr_t bridge_pa; +} t6040_axi2af[] = { + [AXI2AF_ATC0_COMMON] = { "atc0_common", false, 0x390, 0x380000000 }, + [AXI2AF_ATC1_COMMON] = { "atc1_common", false, 0x3b8, 0x388000000 }, + [AXI2AF_ATC2_COMMON] = { "atc2_common", false, 0x3e0, 0x390000000 }, + [AXI2AF_ATC3_COMMON] = { "atc3_common", false, 0x408, 0x398000000 }, + [AXI2AF_DISP_SYS] = { "disp_sys", true, 0x258, 0x403000000 }, + [AXI2AF_DISPEXT0_SYS] = { "dispext0_sys", false, 0x340, 0x3a3000000 }, + [AXI2AF_DISPEXT1_SYS] = { "dispext1_sys", false, 0x490, 0x603000000 }, + [AXI2AF_DISPEXT2_SYS] = { "dispext2_sys", false, 0x4e0, 0x607000000 }, + [AXI2AF_DISPEXT3_SYS] = { "dispext3_sys", false, 0x2d8, 0x307000000 }, + [AXI2AF_ANS] = { "ans", true, 0x370, 0x40b000000 }, + [AXI2AF_AVD_SYS] = { "avd_sys", false, 0x170, 0x280000000 }, + [AXI2AF_VENC0_SYS] = { "venc0_sys", false, 0x278, 0x282000000 }, + [AXI2AF_VENC1_SYS] = { "venc1_sys", false, 0x2a0, 0x284000000 }, + [AXI2AF_JPG] = { "jpg", true, 0x3a0, 0x486000000 }, + [AXI2AF_APCIE_SYS_GP] = { "apcie_sys_gp", true, 0x390, 0x416000000 }, +}; + +static void t6040_pmu_write(struct t6040_sleep *s, u16 reg, u8 val) +{ + apple_sleep_pmu_write(s->spmi0, PMU_USID, reg, val); +} + +/* + * Between the steps of the wake return: every access of a step is complete, and a fault it + * raised is taken, before the next step starts. + */ +static void t6040_wake_sync(void) +{ + dsb(sy); + isb(); +} + +/* + * The five complexes off, measured on the J616s: each slot's low nibble to 0, in order + * +0x0 .. +0x10, and the two display complexes (+0xc, +0x10) polled until BIT4 clears. + * The CPU complexes (E, P0, P1) go down at the WFI. + */ +static void t6040_clusters_off(struct t6040_sleep *s) +{ + u32 val; + int i; + + for (i = 0; i < CLUSTER_CTL_NR; i++) { + val = readl(s->cluster_ctl + CLUSTER_CTL(i)); + s->cluster_saved[i] = val; + writel(val & ~CLUSTER_CTL_NIBBLE, s->cluster_ctl + CLUSTER_CTL(i)); + if (i >= CLUSTER_CTL_FIRST_WAIT) + readl_poll_timeout_atomic(s->cluster_ctl + CLUSTER_CTL(i), val, + !(val & CLUSTER_CTL_BUSY), 1, CLUSTER_TIMEOUT_US); + } +} + +static void t6040_east_clk_quiesce(struct t6040_sleep *s, u32 off) +{ + u32 val = readl(s->soc_east + off), cur; + + writel(val & ~SOC_EAST_CLK_AUTO, s->soc_east + off); + readl_poll_timeout_atomic(s->soc_east + off, cur, + FIELD_GET(PS_ACTUAL, cur) == FIELD_GET(PS_TARGET, val), 1, + CLK_QUIESCE_TIMEOUT_US); +} + +/* The clock quiesce, saving the clk-mux words for wake. */ +static void t6040_clkgen_park(struct t6040_sleep *s) +{ + u32 cur; + int i; + + t6040_east_clk_quiesce(s, SOC_EAST_CLK_A); + t6040_east_clk_quiesce(s, SOC_EAST_CLK_B); + for (i = 0; i < CLKGEN_MUX_WORDS; i++) { + void __iomem *reg = s->clkgen + t6040_clkgen_mux[i]; + u32 val = readl(reg); + + s->clkgen_mux_saved[i] = val; + writel((val & CLKGEN_MUX_MASK) | CLKGEN_MUX_SET, reg); + readl_poll_timeout_atomic(reg, cur, !(cur & CLKGEN_MUX_PENDING), 1, + CLK_QUIESCE_TIMEOUT_US); + } + s->clkgen_saved = true; +} + +/* The mirror on wake: the saved clk-mux words back, last parked first. */ +static void t6040_clkgen_restore(struct t6040_sleep *s) +{ + int i; + + if (!s->clkgen_saved) + return; + for (i = CLKGEN_MUX_WORDS - 1; i >= 0; i--) + writel(s->clkgen_mux_saved[i], s->clkgen + t6040_clkgen_mux[i]); + s->clkgen_saved = false; +} + +/* + * Wait for every other core's power word (SOC + 8 * its cpu-start bit index, + * ECPU_0.. PCPU1_5 in the ADT) to read ACTUAL 0, each for at most 10 ms. + */ +static void t6040_secondaries_off(struct t6040_sleep *s) +{ + u32 self = t6040_cpu_start_bit(read_cpuid_mpidr() & MPIDR_HWID_BITMASK); + unsigned int cpu; + u32 v; + + for_each_present_cpu(cpu) { + u32 bit = t6040_cpu_start_bit(cpu_logical_map(cpu)); + + if (!bit || bit == self) + continue; + readl_poll_timeout_atomic(s->soc_ps + PS_STRIDE * __ffs(bit), v, + !FIELD_GET(PS_ACTUAL, v), 1, 10000); + } +} + +/* The hardware steps of the generic PMGR quiesce, in order (see t6040_acc_apsc_pa). */ +static void t6040_pmgr_quiesce(struct t6040_sleep *s) +{ + u64 q; + u32 v; + int i; + + for (i = 0; i < ARRAY_SIZE(t6040_acc_apsc_pa); i++) { + void __iomem *acc = s->acc_apsc[i]; + + if (!acc) + continue; + writeq(readq(acc) | ACC_APSC_DISABLE, acc); + readq_poll_timeout_atomic(acc, q, !(q & ACC_APSC_BUSY), 1, QUIESCE_TIMEOUT_US); + } + for (i = 0; i < ACC_PLL_RELOCK_COMPLEXES; i++) + if (s->acc_apsc[i]) + writeq(readq(s->acc_apsc[i]) & ~ACC_PLL_RELOCK, s->acc_apsc[i]); + for (i = 0; i < ARRAY_SIZE(t6040_tvm); i++) { + if (!s->tvm_ctl[i] || !s->tvm_status[i]) + continue; + writel(readl(s->tvm_ctl[i]) & ~TVM_ENABLE, s->tvm_ctl[i]); + readl_poll_timeout_atomic(s->tvm_status[i], v, !(v & TVM_BUSY), 1, + QUIESCE_TIMEOUT_US); + } +} + +/* The AGX residency gate of the PMGR quiesce (see T6040_AGX_GATE_PA). */ +static void t6040_agx_gate(struct t6040_sleep *s) +{ + u32 gate, val, cur; + + if (!s->agx_gate || !s->afr_ps) + return; + gate = readl(s->agx_gate); + if (!(~gate & AGX_GATE_RESIDENT)) + return; + val = readl(s->afr_ps); + if (!FIELD_GET(PS_TARGET, val)) + return; + writel((val & ~PS_MIN) | FIELD_PREP(PS_MIN, 4), s->afr_ps); + if (readl_poll_timeout_atomic(s->afr_ps, cur, FIELD_GET(PS_MIN, cur) == 4, 1, + AGX_TIMEOUT_US)) + return; + if (readl_poll_timeout_atomic(s->afr_ps, cur, FIELD_GET(PS_ACTUAL, cur) == 4, 1, + AGX_TIMEOUT_US)) + return; + writel(0, s->afr_ps); + readl_poll_timeout_atomic(s->afr_ps, cur, !FIELD_GET(PS_ACTUAL, cur), 1, AGX_TIMEOUT_US); +} + +/* + * Keep-alive: BIT0 of the 64 keep-alive words, PMGR SOC then SOC-East per + * index. Clears them all (on == false) or sets those in @mask; returns the + * mask of the bits that were set (SOC in the low half, SOC-East in the high). + */ +static u64 t6040_keep_alive(struct t6040_sleep *s, bool on, u64 mask) +{ + u64 was = 0; + int i; + + if (!s->ka_soc || !s->ka_east) + return 0; + for (i = 0; i < KA_WORDS; i++) { + void __iomem *a = s->ka_soc + i * KA_STRIDE; + void __iomem *b = s->ka_east + i * KA_STRIDE; + u32 va = readl(a), vb = readl(b); + + was |= (u64)(va & KA_ENABLE) << i | (u64)(vb & KA_ENABLE) << (i + KA_WORDS); + writel((va & ~KA_ENABLE) | (on && (mask & BIT_ULL(i))), a); + writel((vb & ~KA_ENABLE) | (on && (mask & BIT_ULL(i + KA_WORDS))), b); + } + return was; +} + +/* A domain's power word, from whichever window holds it. */ +static void __iomem *t6040_axi2af_ps(struct t6040_sleep *s, const struct t6040_axi2af_domain *d) +{ + void __iomem *base = d->east ? s->soc_east : s->soc_ps; + + return base ? base + d->ps_off : NULL; +} + +/* The preflight's list of the axi2af domains' power words. */ +static void t6040_axi2af_show(struct t6040_sleep *s, struct seq_file *m) +{ + int i; + + seq_puts(m, "axi2af domains (the quiesce gate wants every one off):\n"); + for (i = 0; i < T6040_AXI2AF_DOMAINS; i++) { + void __iomem *ps = t6040_axi2af_ps(s, &t6040_axi2af[i]); + u32 val = ps ? readl(ps) : 0; + + seq_printf(m, " %-13s %#010x%s\n", t6040_axi2af[i].name, val, + ps ? (FIELD_GET(PS_ACTUAL, val) ? " ON" : "") : " (unmapped)"); + } +} + +static int t6040_sleep_preflight_show(struct seq_file *m, void *unused) +{ + struct t6040_sleep *s = m->private; + unsigned int live = t6040_sleep_live_blockers(s, m, NULL); + static const char * const nm[] = { + "EACC", "PACC0", "PACC1_ANE", "DISP_INT", "DISP_EXT", + }; + unsigned int i; + + apple_sleep_registry_show(m); + t6040_axi2af_show(s, m); + seq_printf(m, "verdict: %u live blocker(s)\n", live); + seq_printf(m, "dry_run %u, wd2_secs %u, allow_entry %d, parking ops %s\n", + READ_ONCE(dry_run), READ_ONCE(wd2_secs), allow_entry, + s->power_ops ? "registered" : "not registered"); + for (i = 0; i < CLUSTER_CTL_NR; i++) + seq_printf(m, "cluster-ctl +%#x %-9s %#x\n", CLUSTER_CTL(i), nm[i], + readl(s->cluster_ctl + CLUSTER_CTL(i))); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(t6040_sleep_preflight); + +/* + * Read-only axi2af gate: every reachable domain's power word is expected to read + * [7:4] == 0 here. The ones still on are only named, at debug level. + */ +static void t6040_axi2af_readcheck(struct t6040_sleep *s) +{ + int i, on = 0; + + for (i = 0; i < T6040_AXI2AF_DOMAINS; i++) { + void __iomem *ps = t6040_axi2af_ps(s, &t6040_axi2af[i]); + u32 val; + + if (!ps) + continue; + val = readl(ps); + if (FIELD_GET(PS_ACTUAL, val)) { + dev_dbg(s->dev, "axi2af %s still on (%#x) at commit\n", + t6040_axi2af[i].name, val); + on++; + } + } + if (on) + dev_dbg(s->dev, "%d axi2af domain(s) still powered at commit\n", on); +} + +/* + * The wake-side axi2af re-arm: every retained domain powered back up (PS TARGET 0xf and + * ACTUAL on) gets its fabric bridge +0x104/+0x108 bit0 set, or the bridge drops its + * traffic. A word in automatic mode can read TARGET 0xf with its domain gated; the bridge + * is then off too, and an access to it raises an SError, so such a domain is left alone. + */ +static void t6040_axi2af_rearm(struct t6040_sleep *s) +{ + int i; + + for (i = 0; i < T6040_AXI2AF_DOMAINS; i++) { + void __iomem *ps = t6040_axi2af_ps(s, &t6040_axi2af[i]); + void __iomem *bridge = s->axi2af_bridge[i]; + u32 val; + + if (!ps || !bridge) + continue; + val = readl(ps); + if (FIELD_GET(PS_TARGET, val) != PS_ACTIVE) + continue; + if (!FIELD_GET(PS_ACTUAL, val)) { + dev_dbg(s->dev, "wake: axi2af %s not re-armed, PS %#x (gated)\n", + t6040_axi2af[i].name, val); + continue; + } + t6040_wake_sync(); + writel(readl(bridge + AXI2AF_BRIDGE_104) | BIT(0), bridge + AXI2AF_BRIDGE_104); + writel(readl(bridge + AXI2AF_BRIDGE_108) | BIT(0), bridge + AXI2AF_BRIDGE_108); + t6040_wake_sync(); + } +} + +/* Arm the 9 AOP-global wake masks before the commit, saving the old words. */ +static void t6040_wake_arm(struct t6040_sleep *s) +{ + int i; + + for (i = 0; i < AOP_WAKE_WORDS; i++) { + s->wake_mask_saved[i] = readl(s->aop_global + AOP_WAKE_MASK(i)); + writel(t6040_wake_mask[i], s->aop_global + AOP_WAKE_MASK(i)); + } + s->wake_mask_saved_valid = true; +} + +/* Put the wake masks back at .end of a returned (or refused) sleep. */ +static void t6040_wake_restore(struct t6040_sleep *s) +{ + int i; + + if (!s->wake_mask_saved_valid) + return; + for (i = 0; i < AOP_WAKE_WORDS; i++) + writel(s->wake_mask_saved[i], s->aop_global + AOP_WAKE_MASK(i)); + s->wake_mask_saved_valid = false; +} + +/* + * One power word raised, as sleep.c's sleep_ps_raise() does: target 0xf with automatic + * mode off and the flag bits cleared, ACTUAL polled, then AUTO_ENABLE. The two halves + * are separate for a word that has to stay at a manual target for a while or for good. + */ +static int t6040_ps_raise_start(void __iomem *base, u32 offset) +{ + return apple_sleep_ps_raise_start(base + offset); +} + +static void t6040_ps_raise_finish(void __iomem *base, u32 offset) +{ + apple_sleep_ps_raise_finish(base + offset); +} + +static int t6040_ps_raise(void __iomem *base, u32 offset) +{ + int ret = t6040_ps_raise_start(base, offset); + + t6040_ps_raise_finish(base, offset); + + return ret; +} + +static bool t6040_ps_actual_on(void __iomem *base, u32 offset) +{ + return FIELD_GET(PS_ACTUAL, readl(base + offset)); +} + +/* The shared PHY and PHY-IP to sleep: the PCIe root complex disable sequence. */ +static void t6040_apcie_phy_off(struct t6040_sleep *s) +{ + struct t6040_apcie_st *a = &s->apcie_st; + void __iomem *ctrl = a->phy + APCIE_PHY_CTRL; + void __iomem *run = a->phy + APCIE_PHY_RUN; + u32 val; + + writel(readl(ctrl) | APCIE_PHY_CTRL_UP, ctrl); + udelay(10); + writel(readl(a->phy_ip + 0x38) & ~GENMASK(27, 26), a->phy_ip + 0x38); + writel(readl(a->phy_ip + 0x8c) | GENMASK(27, 24), a->phy_ip + 0x8c); + udelay(5); + writel(readl(ctrl) | APCIE_PHY_CTRL_RESET, ctrl); + udelay(1); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK1REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK1ACK), 1, APCIE_POLL_US); + udelay(1); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK0REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_POLL_US); + udelay(1); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_UP, ctrl); + udelay(10); + writel(readl(run) & ~APCIE_PHY_RUN_EN2, run); + writel(readl(run) & ~APCIE_PHY_RUN_EN, run); +} + +/* APCIE_PHY_SW to clock-gated, its busy bit polled clear. */ +static void t6040_apcie_phy_sw_gate(struct t6040_sleep *s) +{ + void __iomem *reg = s->soc_east + PS_APCIE_PHY_SW; + u32 val; + + writel(PS_APCIE_PHY_SW_CLKGATED, reg); + readl_poll_timeout_atomic(reg, val, !(val & PS_BUSY), 1, PS_OFF_TIMEOUT_US); +} + +/* The PCIe PLL shutdown: the clock generator's control word. */ +static void t6040_apcie_pll_off(struct t6040_sleep *s) +{ + void __iomem *clk = s->apcie_st.clkgen + APCIE_CLK_CTRL; + u32 val; + + writel((readl(clk) & ~(APCIE_CLK_RUN | GENMASK(4, 3))) | GENMASK(4, 3), clk); + writel(readl(clk) | APCIE_CLK_OFF, clk); + writel(readl(clk) & ~BIT(0), clk); + readl_poll_timeout_atomic(clk, val, !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + writel(readl(clk) | APCIE_CLK_OFF2, clk); +} + +/* + * The sleep engine hands each ATC port to its always-on side and then powers the port's + * common domain off itself: it needs the four ATC commons on at that + * point. With them off it waits for good raising ATC0_COMMON, whose parent FAB2_SOC is + * off as well. FAB2_SOC is raised by hand, then ATC0..3_COMMON to a manual target 0xf, + * then FAB2_SOC gets its automatic mode back: the powered children hold it up. + */ +static const u16 t6040_atc_common[4] = { 0x390, 0x3b8, 0x3e0, 0x408 }; + +static void t6040_atc_commons_on(struct t6040_sleep *s) +{ + void __iomem *b = s->soc_ps; + int i; + + t6040_ps_raise_start(b, PS_FAB2_SOC); + for (i = 0; i < ARRAY_SIZE(t6040_atc_common); i++) + t6040_ps_raise_start(b, t6040_atc_common[i]); + t6040_ps_raise_finish(b, PS_FAB2_SOC); +} + +/* + * The CFG writes of the PHY sleep entry, run on every ATC port before a sleep: + * ATCPHY_CFG +0x2c bit 0 (S2R_AON_SYNC_RST_N), +0x08 bit 0 (CFG_S2R_FSM_EN) and +0x28 bit 0 + * (AHB_CLK_REQ_N) set, then +0x04 bit 8 (the clock's ack) polled. Without them the engine + * waits for good on a port that carries DisplayPort lanes: port 3 (the HDMI output) always, + * any other port once a display runs on it in DP alternate mode. A parked port, and + * one its PHY driver has put to sleep, already has +0x28 bit 0 set and is left alone. + */ +#define ATC_CFG_CLK_STATUS 0x04 +#define ATC_CFG_CLK_ACK BIT(8) +#define ATC_CFG_S2R_FSM 0x08 +#define ATC_CFG_S2R_FSM_EN BIT(0) +#define ATC_CFG_AHB_CLK 0x28 +#define ATC_CFG_AHB_CLK_REQ_N BIT(0) +#define ATC_CFG_S2R_AON 0x2c +#define ATC_CFG_S2R_AON_SYNC_RST_N BIT(0) + +static void t6040_atc_s2r_entry(struct t6040_sleep *s) +{ + u32 v; + int n; + + for (n = 0; n < ARRAY_SIZE(s->atc_cfg); n++) { + void __iomem *c = s->atc_cfg[n]; + + if (!c || (readl(c + ATC_CFG_AHB_CLK) & ATC_CFG_AHB_CLK_REQ_N)) + continue; + writel(readl(c + ATC_CFG_S2R_AON) | ATC_CFG_S2R_AON_SYNC_RST_N, + c + ATC_CFG_S2R_AON); + udelay(1); + writel(readl(c + ATC_CFG_S2R_FSM) | ATC_CFG_S2R_FSM_EN, c + ATC_CFG_S2R_FSM); + udelay(5); + writel(readl(c + ATC_CFG_AHB_CLK) | ATC_CFG_AHB_CLK_REQ_N, c + ATC_CFG_AHB_CLK); + readl_poll_timeout_atomic(c + ATC_CFG_CLK_STATUS, v, v & ATC_CFG_CLK_ACK, 1, 1000); + } +} + +/* + * The 13 fabric power words. The fabrics are expected up through the entry, and the wake + * polls these words for automatic mode with target 0xf. Under Linux genpd has powered some + * of them off with their last consumer by now (FAB2_SOC with the ATC commons at + * suspend_noirq). Right before the S2R timer each one not in that state is raised: target + * 0xf, ACTUAL 0xf polled, then automatic mode. The ATC commons and the ports' PHY sleep + * entry follow, in this order. + */ +static const struct { + bool east; + u16 off; +} t6040_fabrics[13] = { + { false, 0x110 } /* AFI */, { false, 0x178 } /* AFC */, + { false, 0x448 } /* SBR */, { false, 0x100 } /* PMS */, + { false, 0x168 } /* FAB0_SOC */, { false, 0x2d0 } /* FAB1_SOC */, + { false, 0x338 } /* FAB2_SOC */, { false, 0x440 } /* FAB5_SOC */, + { false, 0x480 } /* FAB6_SOC */, { false, 0x488 } /* FAB7_SOC */, + { true, 0x118 } /* FAB3_SOC */, { true, 0x128 } /* FAB4_SOC */, + { false, 0x298 } /* FAB_AFR */, +}; + +static void t6040_fabrics_on(struct t6040_sleep *s) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(t6040_fabrics); i++) { + void __iomem *b = t6040_fabrics[i].east ? s->soc_east : s->soc_ps; + u32 off = t6040_fabrics[i].off, v; + + v = readl(b + off); + if (!(v & PS_AUTO_ENABLE) || FIELD_GET(PS_TARGET, v) != PS_ACTIVE) + t6040_ps_raise(b, off); + } + t6040_atc_commons_on(s); + t6040_atc_s2r_entry(s); +} + +/* + * The memory channels' power gating. The sleep engine does not write the DCS power words: it + * expects each DCS to gate itself after self-refresh. That takes the DCS's dynamic power-gate + * mode (DCS +0x20 bits 12 and 8) and its power word in automatic mode with no minimum state. + * Both are expected at boot and again after each wake. Nothing sets them under Linux (the DT leaves + * the ps_dcs_* domains disabled: target 0xf, no automatic mode), and the engine then waits + * for the first DCS pair's power gate for good. Both are applied in each .begin. + */ +static const struct { + bool east; + u16 off; +} t6040_dcs_ps[16] = { + { false, 0x188 }, { false, 0x1a0 }, { false, 0x1b0 }, { false, 0x1c0 }, + { true, 0x2e0 }, { true, 0x2e8 }, { true, 0x2f0 }, { true, 0x2f8 }, + { false, 0x1d0 }, { false, 0x1e0 }, { false, 0x1f0 }, { false, 0x200 }, + { true, 0x300 }, { true, 0x308 }, { true, 0x310 }, { true, 0x318 }, +}; + +static void __iomem *t6040_dcs_ps_reg(struct t6040_sleep *s, int i) +{ + void __iomem *b = t6040_dcs_ps[i].east ? s->soc_east : s->soc_ps; + + return b + t6040_dcs_ps[i].off; +} + +/* + * In order: the minimum state cleared, target 0xf, ACTUAL 0xf polled, then automatic + * mode. Returns a mask of the DCS whose ACTUAL did not reach 0xf. + */ +static u32 t6040_dcs_ps_auto_on(struct t6040_sleep *s) +{ + u32 to = 0, v; + int i; + + for (i = 0; i < ARRAY_SIZE(t6040_dcs_ps); i++) { + void __iomem *r = t6040_dcs_ps_reg(s, i); + + v = readl(r); + if ((v & PS_AUTO_ENABLE) && !(v & PS_MIN)) + continue; + writel((v & ~(PS_MIN | PS_FLAGS | PS_TARGET)) | + FIELD_PREP(PS_TARGET, PS_ACTIVE), r); + if (readl_poll_timeout_atomic(r, v, FIELD_GET(PS_ACTUAL, v) == PS_ACTIVE, 1, + PS_OFF_TIMEOUT_US)) + to |= BIT(i); + writel(readl(r) | PS_AUTO_ENABLE, r); + } + return to; +} + +#define DCS_PG_CTL 0x20 +#define DCS_PG_DYN (BIT(12) | BIT(8)) + +static void t6040_dcs_dyn_pg(struct t6040_sleep *s) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(s->dcs); i++) { + u32 v; + + if (!s->dcs[i]) + continue; + v = readl(s->dcs[i] + DCS_PG_CTL); + if ((v & DCS_PG_DYN) != DCS_PG_DYN) + writel(v | DCS_PG_DYN, s->dcs[i] + DCS_PG_CTL); + } +} + +/* + * ANS's own fabric bridge (0x4_0b00_0000 +0x104/+0x108 bit 0). The bit is cleared before a + * retained domain goes off and set again once it is back up; here the wake sets it + * (t6040_apcie_st_raise()) and the SoC's sleep engine, not this driver, takes ANS down. + * The boot loader leaves the bit clear, so a boot's first sleep goes down with it clear; + * without this the next one meets the bridge as the first wake armed it. Only while ANS is + * on: the bridge sits in it. + */ +static void t6040_ans_bridge_quiesce(struct t6040_sleep *s) +{ + void __iomem *bridge = s->axi2af_bridge[AXI2AF_ANS]; + u32 a, b; + + if (!bridge || + FIELD_GET(PS_ACTUAL, readl(s->soc_east + PS_ANS)) != PS_ACTIVE) + return; + a = readl(bridge + AXI2AF_BRIDGE_104); + b = readl(bridge + AXI2AF_BRIDGE_108); + if (!((a | b) & BIT(0))) + return; + writel(a & ~BIT(0), bridge + AXI2AF_BRIDGE_104); + writel(b & ~BIT(0), bridge + AXI2AF_BRIDGE_108); +} + +/* + * Quiesce ANS's storage link, once ANS has been stopped by its driver. The PHY, the + * clock generator and the AXI enables are treated as shared by the two ST lanes. + * + * The power words of this group are left to the SoC's sleep engine, which powers it down + * itself and in its own order: APCIE_PHY_SW to clock-gated, APCIE_SYS_ST0/1 off, APCIE_ST0/1 + * clock-gated, ANS off, APCIE_SYS_GP off, APCIE_GP clock-gated. It polls APCIE_PHY_SW for + * its parents, which are both SYS_ST lanes: with them lowered here, as the T8140 controller + * does, it waits there for good. So the commit raises APCIE_SYS_ST1, which + * Linux never uses, and does only what the engine needs before it runs: the shared PHY put + * to sleep, APCIE_PHY_SW clock-gated while APCIE_GP is still on, the PCIe PLL shut down. + */ +static void t6040_apcie_st_lower(struct t6040_sleep *s) +{ + struct t6040_apcie_st *a = &s->apcie_st; + void __iomem *ps = s->soc_east; + + if (!a->valid || !ps) + return; + + /* The PHY's registers need a lane domain on; quiet it first (shared). */ + if (a->phy && t6040_ps_actual_on(ps, PS_APCIE_SYS_ST0)) + t6040_apcie_phy_off(s); + t6040_apcie_phy_sw_gate(s); + if (a->clkgen && t6040_ps_actual_on(ps, PS_APCIE_SYS_ST0)) + t6040_apcie_pll_off(s); + + /* AXI enables off (shared). */ + if (a->axi) { + writel(readl(a->axi + APCIE_AXI_EN0) & ~BIT(0), a->axi + APCIE_AXI_EN0); + writel(readl(a->axi + APCIE_AXI_EN1) & ~BIT(0), a->axi + APCIE_AXI_EN1); + } + t6040_ans_bridge_quiesce(s); +} + +static void t6040_bridge_rearm(void __iomem *bridge) +{ + if (!bridge) + return; + writel(readl(bridge + AXI2AF_BRIDGE_104) | BIT(0), bridge + AXI2AF_BRIDGE_104); + writel(readl(bridge + AXI2AF_BRIDGE_108) | BIT(0), bridge + AXI2AF_BRIDGE_108); +} + +/* + * Raise ANS's storage link from power-off, the two-lane adaptation of + * sleep.c's apple_sleep_raise_apcie(). Runs atomic on the waking core in + * t6040_sleep_return(), after t6040_axi2af_rearm() and before control returns to + * the PM core, so nvme's resume_early sets the ANS run bit onto a live link. + * Both lanes come up; the PLL/clkgen/PHY tables run once (shared). The two + * apcie_sys_st domains are not in the 15-domain axi2af table, so their bridges + * get no re-arm. + * + * The AXI enables (+0x104/+0x108 of 0x4_1600_0000) are the fabric bridge of + * APCIE_SYS_GP, which the commit lowered together with APCIE_GP: both are raised + * here before the enables are written. APCIE_PHY_SW, which the commit gated, + * comes back before the PHY, and the PHY's reference PLL (common +0 bit 31) is + * waited for before the PHY's clocks are requested. @apcie_wake_step tells a + * failed raise's stage. + */ +static int t6040_apcie_st_raise(struct t6040_sleep *s) +{ + struct t6040_apcie_st *a = &s->apcie_st; + void __iomem *ps = s->soc_east; + void __iomem *ctrl, *run; + u32 val; + int ret; + + if (!a->valid || !ps) + return 0; + ctrl = a->phy + APCIE_PHY_CTRL; + run = a->phy + APCIE_PHY_RUN; + + /* + * In order, each domain to ACTUAL 0xf: ANS, GP, SYS_GP, ST0, SYS_ST0, ST1, SYS_ST1, + * the fabric-bridge bit0s set after ANS and after SYS_GP. SYS_GP stays at a manual + * target 0xf: in automatic mode it gates again, and the DART behind it then faults in + * its resume. + */ + ret = t6040_ps_raise(ps, PS_ANS); + if (!ret) + t6040_bridge_rearm(s->axi2af_bridge[AXI2AF_ANS]); + ret |= t6040_ps_raise(ps, PS_APCIE_GP); + if (!ret) + ret = t6040_ps_raise_start(ps, PS_APCIE_SYS_GP); + if (!ret) { + writel(readl(a->axi + APCIE_AXI_EN0) | BIT(0), a->axi + APCIE_AXI_EN0); + writel(readl(a->axi + APCIE_AXI_EN1) | BIT(0), a->axi + APCIE_AXI_EN1); + } + ret |= t6040_ps_raise(ps, PS_APCIE_ST0); + ret |= t6040_ps_raise(ps, PS_APCIE_SYS_ST0); + ret |= t6040_ps_raise(ps, PS_APCIE_ST1); + ret |= t6040_ps_raise(ps, PS_APCIE_SYS_ST1); + t6040_wake_sync(); + s->apcie_wake_step = 1; + if (ret) + return ret; + + /* Tunables + clkgen (shared). Both OFF bits clear; only a power-off cleared them. */ + apple_tunable_apply(a->axi, a->axi_tunable); + apple_tunable_apply(a->cio3pll, a->cio3pll_tunable); + apple_tunable_apply(a->clkgen, a->clkgen_tunable); + writel(readl(a->clkgen + APCIE_CLK_START) | APCIE_CLK_START_GO, + a->clkgen + APCIE_CLK_START); + writel(readl(a->clkgen + APCIE_CLK_CTRL) & ~(APCIE_CLK_OFF2 | APCIE_CLK_OFF), + a->clkgen + APCIE_CLK_CTRL); + ret = readl_poll_timeout_atomic(a->clkgen + APCIE_CLK_CTRL, val, + !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + t6040_wake_sync(); + s->apcie_wake_step = 2; + if (ret) + return ret; + + ret = t6040_ps_raise(ps, PS_APCIE_PHY_SW); + s->apcie_wake_step = 3; + if (ret) + return ret; + + /* Shared PHY bring-up + PLL/lane tables (as sleep.c's sleep_apcie_phy_ip_apply()). */ + apple_tunable_apply(a->phy, a->phy_tunable); + ret = readl_poll_timeout_atomic(a->phy + APCIE_PHY_COMMON, val, val & BIT(31), 1, + APCIE_POLL_US); + t6040_wake_sync(); + s->apcie_wake_step = 4; + if (ret) + return ret; + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK0REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK0ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK1REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK1ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) & ~APCIE_PHY_CTRL_RESET, ctrl); + t6040_wake_sync(); + s->apcie_wake_step = 5; + + writel(readl(run) | APCIE_PHY_RUN_EN, run); + apple_sleep_tunable_apply_changed(a->phy_ip, a->phy_ip_pll); + apple_sleep_tunable_apply_changed(a->phy_ip, a->phy_ip_lanes); + writel(readl(a->phy_ip + APCIE_PHY_IP_PLL_CTL) & ~APCIE_PHY_IP_PLL_HOLD, + a->phy_ip + APCIE_PHY_IP_PLL_CTL); + writel(readl(run) | APCIE_PHY_RUN_EN2, run); + ret = readl_poll_timeout_atomic(a->phy + APCIE_PHY_READY, val, val & BIT(0), 1, + APCIE_POLL_US); + if (ret) + return ret; + t6040_wake_sync(); + s->apcie_wake_step = 6; + writel(readl(a->phy + APCIE_PHY_COMMON) | BIT(0), a->phy + APCIE_PHY_COMMON); + writel(readl(ctrl) | APCIE_PHY_CTRL_UP, ctrl); + t6040_wake_sync(); + s->apcie_wake_step = 7; + + /* RC ready handshake (as T8140's). */ + writel(0, a->rc + APCIE_RC_ZERO); + writel(0x140, a->rc + APCIE_RC_READY_CFG); + writel(1, a->rc + APCIE_RC_READY_START); + return readl_poll_timeout_atomic(a->rc + APCIE_RC_READY_STAT, val, val & BIT(0), 1, + APCIE_POLL_US); +} + +/* + * The first thing Linux runs after a wake, MMU off and no stack (as the T8132 stub does). The + * last wake program leaves the AON WD2 running for 5 s, less than the device resume takes: + * restart it for 60 s, in t6040_sleep_set_wd2()'s order, then take cpu_resume. The PA is + * the T6040 AON WDT (T6040_WDT_PA), used only when the DT's wdt block is that one. + */ +asm(" .pushsection .text, \"ax\"\n" +" .type t6040_resume_stub, %function\n" +"t6040_resume_stub:\n" +" movz x9, #0xc000\n" +" movk x9, #0x0836, lsl #16\n" +" movk x9, #0x5, lsl #32\n" +" str wzr, [x9, #0x2c]\n" +" str wzr, [x9, #0x20]\n" +" movz w10, #0xa800\n" +" movk w10, #0x55d4, lsl #16\n" +" str w10, [x9, #0x28]\n" +" mov w10, #1\n" +" str w10, [x9, #0x2c]\n" +" b cpu_resume\n" +" .size t6040_resume_stub, . - t6040_resume_stub\n" +" .popsection\n" +); +extern char t6040_resume_stub[]; + +/* + * The four ATC complexes' power words in SOC (soc_ps, 0x502800000), deepest first: cio_usb, + * cio_pcie, cio, pcie, common (ADT: ATCn_CIO_USB .. ATCn_COMMON). The quiesce gate wants + * them off; their drivers don't lower atcN_common. + * + * Their common_dp, usb, phymxwrap and usb_aon words are MiniPMGR words (0x508300000 + 0xa0.., + * ADT ATCn_COMMON_DP/USB/PHYMXWRAP/USB_AON) and not in this table: the same offsets in SOC + * are AIC, PMC, PMS_FPWM0-4 and FAB0_SOC. + */ +static const u16 t6040_atc_words[4][5] = { + { 0x3b0, 0x3a8, 0x398, 0x3a0, 0x390 }, + { 0x3d8, 0x3d0, 0x3c0, 0x3c8, 0x3b8 }, + { 0x400, 0x3f8, 0x3e8, 0x3f0, 0x3e0 }, + { 0x438, 0x430, 0x420, 0x428, 0x408 }, +}; + +/* A domain lowered (T8140 measured): AUTO_ENABLE off first, then target 0, poll. */ +static int t6040_ps_lower_word(void __iomem *reg) +{ + u32 val = readl(reg); + + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + writel((val & ~PS_TARGET) | PS_FLAGS, reg); + return readl_poll_timeout_atomic(reg, val, !FIELD_GET(PS_ACTUAL, val), 1, + PS_OFF_TIMEOUT_US); +} + +static void t6040_atc_lower(struct t6040_sleep *s) +{ + int p, k; + + for (p = 0; p < ARRAY_SIZE(t6040_atc_words); p++) + for (k = 0; k < ARRAY_SIZE(t6040_atc_words[0]); k++) { + void __iomem *reg = s->soc_ps + t6040_atc_words[p][k]; + u32 val = readl(reg); + + if (!FIELD_GET(PS_TARGET, val) && !FIELD_GET(PS_ACTUAL, val)) + continue; + t6040_ps_lower_word(reg); + } +} + +/* + * T8140's radio port stop, on the same t602x port block: the link into L2, the endpoint's + * PERST# asserted, the PHY's clocks off. The endpoint keeps its power and comes back by its + * host after the sleep. + */ +static void t6040_gp_port_stop(struct t6040_sleep *s, int p) +{ + void __iomem *port = s->gp_port[p]; + void __iomem *phy = s->gp_phy[p]; + u32 val; + + if (!port || !phy) + return; + s->gp_phy_saved[p] = readl(phy); + s->gp_appclk_saved[p] = readl(port + APCIE_PORT_APPCLK); + s->gp_perst_saved[p] = readl(port + APCIE_PORT_PERST); + s->gp_saved[p] = true; + writel(readl(port + APCIE_PORT_INTMSK) | APCIE_PORT_INT_LINK, port + APCIE_PORT_INTMSK); + writel(APCIE_PORT_INT_LINK, port + APCIE_PORT_INTSTAT); + writel(readl(port + APCIE_PORT_APPCLK) | APCIE_PORT_APPCLK_CGDIS, port + APCIE_PORT_APPCLK); + if (readl(port + APCIE_PORT_LINKSTS) & APCIE_PORT_LINKSTS_UP) { + writel(APCIE_PORT_L2_ENTER, port + APCIE_PORT_L2); + readl_poll_timeout_atomic(port + APCIE_PORT_L2, val, !(val & APCIE_PORT_L2_BUSY), + 10, APCIE_PORT_STOP_US); + } + if (s->gp_pins && s->gp_perst[p] >= 0) { + void __iomem *reg = s->gp_pins + 4 * s->gp_perst[p]; + + val = readl(reg); + writel(s->gp_perst_low[p] ? val & ~GP_PIN_DATA : val | GP_PIN_DATA, reg); + } + writel(readl(phy) & ~APCIE_PORT_PHY_LINK, phy); + udelay(1000); + /* What this bit does isn't known. It is set around PERST# (T8140). */ + writel(readl(port + APCIE_PORT_PERST_GUARD) | BIT(8), port + APCIE_PORT_PERST_GUARD); + writel(0, port + APCIE_PORT_PERST); + writel(readl(port + APCIE_PORT_PERST_GUARD) & ~BIT(8), port + APCIE_PORT_PERST_GUARD); + writel(readl(phy) & ~APCIE_PORT_PHY_EN1, phy); + writel(readl(phy) & ~APCIE_PORT_PHY_EN0, phy); + writel(readl(phy) | APCIE_PHY_CTRL_RESET, phy); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK0REQ, phy); + readl_poll_timeout_atomic(phy, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_PORT_STOP_US); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK1REQ, phy); + writel(readl(port + APCIE_PORT_APPCLK) & ~APCIE_PORT_APPCLK_EN, port + APCIE_PORT_APPCLK); + writel(readl(port + APCIE_PORT_INTSTAT), port + APCIE_PORT_INTSTAT); +} + +/* Whether the GP host's system domain is still on; if not, there is nothing left to stop. */ +static __always_inline bool t6040_apcie_sys_gp_on(struct t6040_sleep *s) +{ + return FIELD_GET(PS_ACTUAL, readl(s->soc_east + PS_APCIE_SYS_GP)); +} + +/* The GP host's two ports, while the block's domains are all still on. */ +static void t6040_gp_ports_stop(struct t6040_sleep *s) +{ + int p; + + if (!t6040_apcie_sys_gp_on(s)) + return; + for (p = 0; p < ARRAY_SIZE(s->gp_port); p++) + t6040_gp_port_stop(s, p); +} + +/* + * apcie_sys_gp (AUTO off, axi2af bit0s, target 0), then apcie_gp. After ANS's ST lanes: the + * storage lanes' cio3pll/clkgen/PHY windows sit in apcie_gp, and an access to them with it + * off hangs the bus. T8140's order: SYS_ST, ST, SYS_GP, GP. APCIE_GP goes to clock-gated + * (TARGET 4), the state the sleep engine itself asks for, not to off. + */ +static void t6040_apcie_gp_lower(struct t6040_sleep *s) +{ + void __iomem *sys, *gp, *bridge = s->axi2af_bridge[AXI2AF_APCIE_SYS_GP]; + u32 val; + + sys = s->soc_east + PS_APCIE_SYS_GP; + gp = s->soc_east + PS_APCIE_GP; + if (!t6040_apcie_sys_gp_on(s)) + return; + + val = readl(sys); + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, sys); + } + if (bridge) { + writel(readl(bridge + AXI2AF_BRIDGE_104) & ~BIT(0), bridge + AXI2AF_BRIDGE_104); + writel(readl(bridge + AXI2AF_BRIDGE_108) & ~BIT(0), bridge + AXI2AF_BRIDGE_108); + } + writel((val & ~PS_TARGET) | PS_FLAGS, sys); + readl_poll_timeout_atomic(sys, val, !FIELD_GET(PS_ACTUAL, val), 1, PS_OFF_TIMEOUT_US); + + writel(PS_APCIE_GP_CLKGATED, gp); + readl_poll_timeout_atomic(gp, val, FIELD_GET(PS_ACTUAL, val) == 4, 1, PS_OFF_TIMEOUT_US); +} + +/* + * The real commit (t6040_sleep_commit, reached through cpu_suspend): t6040_sleep_enter + * calls it only when no blocker remains and allow_entry is set. In order: the stopped CPUs + * released to their last WFI, the ATC complexes and the GP PCIe ports lowered, ANS's storage + * link quiesced, WD0 off and the AON WD2 net armed, the PMU's boot stage 0x50 and + * NOTIFY_ENABLE over spmi/spmi-a0, the steps of the PMGR quiesce (APSC, PLL relock, + * TVMs, the AGX gate, the five complexes, the axi2af read-check, the clock quiesce, the + * keep-alive bits), aura's mailbox armed with the resume entry, the fabrics and the ATC + * ports readied for the sleep engine, the MasterMini S2R timer (0x508904000 := 1), the AIC + * masked and the deep WFI. The watchdog, PMU, cluster, axi2af and timer writes are as + * measured on the J616s. A hang here ends at WD2. + */ +static int t6040_sleep_commit(unsigned long unused) +{ + struct t6040_sleep *s = t6040_sleep; + + s->siq_cfg = read_sysreg_s(SYS_APL_SIQ_CFG_EL1); + spin_table_stop_parked(); + /* + * Every other core has to be off, in its last WFI, before the boot core turns the + * complexes off, or the sleep engine waits for them: release them now, and wait for + * their power words to read off further down. + */ + spin_table_release_stopped(); + t6040_atc_lower(s); + t6040_gp_ports_stop(s); + + /* + * ANS's storage link, once ANS has been stopped by its driver (see + * t6040_apcie_phy_off). It follows every refusal, so a refused sleep finds + * the link up. + */ + t6040_ps_raise_start(s->soc_east, PS_APCIE_SYS_ST1); + if (s->pcie_ans_stopped) + t6040_apcie_st_lower(s); + t6040_apcie_gp_lower(s); + + /* WD0 off, then the AON WD2 net (5 s covers the reconfig). */ + writel(0, s->wdt + WDT_WD0_CTL); + t6040_sleep_set_wd2(s, clamp(READ_ONCE(sleep_wd_secs), 1U, WDT_MAX_SECS)); + /* And the SMC's AP watchdog deadline off, right after the WD2 (measured). */ + if (s->smc_wdt_deadline) + writel(0, s->smc_wdt_deadline); + + /* PMU boot stage 0x50 and NOTIFY_ENABLE on both SPMI controllers (measured). */ + t6040_pmu_write(s, PMU_BOOT_STAGE, PMU_BOOT_STAGE_SLEEP); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi0 + SPMI_NOTIFY_ENABLE); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi_a0 + SPMI_NOTIFY_ENABLE); + + /* + * The order of the PMGR quiesce (the cluster and axi2af + * parts measured on the J616s): the five complexes off, the axi2af gate read, + * the clock quiesce, the keep-alive bits off, then the S2R timer. + */ + t6040_secondaries_off(s); + t6040_pmgr_quiesce(s); + t6040_agx_gate(s); + t6040_clusters_off(s); + + /* Read-only: the axi2af panic gate checked before the engine goes. */ + t6040_axi2af_readcheck(s); + t6040_clkgen_park(s); + s->ka_on_before = t6040_keep_alive(s, false, 0); + + /* Arm aura's AURAS2R1 mailbox with this boot's resume entry. */ + if (s->mbox) { + WRITE_ONCE(s->mbox->resume_pa, + s->wdt_stub_ok ? + __pa_symbol(t6040_resume_stub) : __pa_symbol(cpu_resume)); + dcache_clean_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + dsb(sy); + } + + t6040_fabrics_on(s); + t6040_sio_ps_save(s); + + /* The commit: the MasterMini S2R timer arms the SoC sleep engine. */ + writel(1, s->mastermini + MASTERMINI_S2R_TIMER); + dsb(sy); + + /* A raised line would end the WFI the engine waits for. */ + aic_sleep_mask_all(); + spin_table_release_stopped(); + + t6040_deep_wfi(); +} + +/* + * The GP PCIe host's two ports (Wi-Fi/BT, SD): the boot loader set their clocks up, and + * nobody does after a sleep. Without them the DART in port 0's block faults in its noirq + * resume. t6040_gp_port_stop() backwards, short of the link: the lane reference clocks + * requested and acknowledged, the PHY's enables and the port's application clock as saved, + * the controller's PERST register as saved, then the port's ready bit polled. The link + * itself stays down for the host driver. + */ +static void t6040_gp_pcie_clk(struct t6040_sleep *s) +{ + const u32 appclk = APCIE_PORT_APPCLK_EN | APCIE_PORT_APPCLK_CGDIS; + unsigned int timeouts = 0; + u32 val; + int i; + + for (i = 0; i < ARRAY_SIZE(s->gp_port); i++) { + void __iomem *phy = s->gp_phy[i], *port = s->gp_port[i]; + u32 cfg, app; + + if (!phy || !port || !s->gp_saved[i]) + continue; + cfg = s->gp_phy_saved[i]; + app = s->gp_appclk_saved[i]; + writel(readl(phy) | APCIE_PHY_CTRL_CLK0REQ, phy); + if (readl_poll_timeout_atomic(phy, val, val & APCIE_PHY_CTRL_CLK0ACK, 1, 50000)) + timeouts++; + writel(readl(phy) | APCIE_PHY_CTRL_CLK1REQ, phy); + if (readl_poll_timeout_atomic(phy, val, val & APCIE_PHY_CTRL_CLK1ACK, 1, 50000)) + timeouts++; + if (!(cfg & APCIE_PHY_CTRL_RESET)) { + writel(readl(phy) & ~APCIE_PHY_CTRL_RESET, phy); + udelay(1); + } + writel(readl(phy) | (cfg & (APCIE_PORT_PHY_EN0 | APCIE_PORT_PHY_EN1)), phy); + writel(readl(phy) | (cfg & APCIE_PORT_PHY_LINK), phy); + writel((readl(port + APCIE_PORT_APPCLK) & ~appclk) | (app & appclk), + port + APCIE_PORT_APPCLK); + /* + * The stop also wrote the controller's PERST register 0, and without it the + * port does not report ready. Put it back as it was (the endpoint's PERST# pin + * stays asserted), with the guard bit set around it. + */ + writel(readl(port + APCIE_PORT_PERST_GUARD) | BIT(8), + port + APCIE_PORT_PERST_GUARD); + writel(s->gp_perst_saved[i], port + APCIE_PORT_PERST); + writel(readl(port + APCIE_PORT_PERST_GUARD) & ~BIT(8), + port + APCIE_PORT_PERST_GUARD); + if (readl_poll_timeout_atomic(port + APCIE_PORT_STATUS, val, + val & APCIE_PORT_STATUS_READY, 1, 50000)) + timeouts++; + } + s->gp_clk_timeouts = timeouts; +} + +/* + * The wake return, on the boot core, once the boot object's mailbox has handed it to the + * resume entry. The boot object restores no SoC register: every write here is this driver's. + * In order: the timer and IRQ routing, the stopped CPUs and the AIC, then as the PMGR + * wake restore the complexes back, the two wake pwrgates, SOC-East retention off (with the + * busy poll) and the axi2af fabric-bridge re-arm; what a Linux sleep needs on top, ANS's + * storage link, the GP PCIe ports' clocks and the AOP island's power words; then the CPM + * voters re-asserted, the clk mux restored, the AOP-global S2R control cleared, the + * keep-alive bits back, WD2 armed over the device resume, and the wake-status words read. + */ +static void t6040_sleep_return(struct t6040_sleep *s) +{ + u32 val; + int i, ret; + + /* The reset left the timer and IRQ routing at their EL2 defaults. */ + write_sysreg(s->cntkctl, cntkctl_el1); + isb(); + write_sysreg_s(s->siq_cfg, SYS_APL_SIQ_CFG_EL1); + t6040_wake_sync(); + + spin_table_stop_finished(); + t6040_wake_sync(); + aic_sleep_restore(); + t6040_wake_sync(); + + /* + * The complexes back: E, P0 and P1 to 0xf, the two display complexes to the nibble + * the commit found, with no poll. Register 0xe440f8 of each CPU complex could also + * be written 1. That is not done here: the secondaries come up through the + * normal per-CPU start afterwards, and all 14 CPUs of the J616s are + * online after a wake without it. + */ + for (i = 0; i < CLUSTER_CTL_NR; i++) { + u32 on = i < CLUSTER_CTL_FIRST_WAIT ? CLUSTER_CTL_ON : + s->cluster_saved[i] & CLUSTER_CTL_NIBBLE; + + val = readl(s->cluster_ctl + CLUSTER_CTL(i)); + writel((val & ~CLUSTER_CTL_NIBBLE) | on, s->cluster_ctl + CLUSTER_CTL(i)); + } + t6040_wake_sync(); + + /* Wake pwrgates and CPM voters: read, set BIT31. */ + writel(readl(s->pwrgate + PWRGATE_WAKE_0) | PWRGATE_ON, s->pwrgate + PWRGATE_WAKE_0); + writel(readl(s->pwrgate + PWRGATE_WAKE_1) | PWRGATE_ON, s->pwrgate + PWRGATE_WAKE_1); + t6040_wake_sync(); + + /* SOC-East retention off: clear DEV_DISABLE, then poll BUSY clear. */ + val = readl(s->soc_east + SOC_EAST_RETENTION); + writel(val & ~PS_DEV_DISABLE, s->soc_east + SOC_EAST_RETENTION); + ret = readl_poll_timeout_atomic(s->soc_east + SOC_EAST_RETENTION, val, + !(val & PS_BUSY), 1, SOC_EAST_TIMEOUT_US); + if (ret) + dev_warn(s->dev, "SOC-East retention word still busy on wake (%#x)\n", val); + t6040_wake_sync(); + + /* + * Bit 0 of the SoC's 14 energy accumulators could be set again here. They + * only account energy and nothing in this kernel reads them, so they are + * left as the power loss left them. + */ + + /* The fabric-bridge retention re-arm, or the bridges drop their traffic. */ + t6040_axi2af_rearm(s); + t6040_wake_sync(); + + /* ANS's storage link, after the bridges and before nvme's resume_early (see there). */ + s->apcie_wake_step = 0; + s->pcie_ans_on_ret = 0; + if (s->pcie_ans_stopped) + s->pcie_ans_on_ret = t6040_apcie_st_raise(s); + t6040_gp_pcie_clk(s); + t6040_wake_sync(); + t6040_aop_ps_restore(s); + t6040_wake_sync(); + + writel(readl(s->pwrgate + PWRGATE_CPM_0) | PWRGATE_ON, s->pwrgate + PWRGATE_CPM_0); + writel(readl(s->pwrgate + PWRGATE_CPM_1) | PWRGATE_ON, s->pwrgate + PWRGATE_CPM_1); + writel(readl(s->pwrgate + PWRGATE_CPM_2) | PWRGATE_ON, s->pwrgate + PWRGATE_CPM_2); + t6040_wake_sync(); + + /* Undo the entry clk-mux park; the clock restore is otherwise a NOP. */ + t6040_clkgen_restore(s); + t6040_wake_sync(); + + /* Software ack of the armed S2R request (the S2R timer has no stop). */ + writel(0, s->aop_global + AOP_S2R_CTRL); + t6040_wake_sync(); + + /* The keep-alive bits the commit cleared, back (the wake restore enables them). */ + if (s->ka_on_before) + t6040_keep_alive(s, true, s->ka_on_before); + t6040_wake_sync(); + + /* A longer WD2 over the device resume, until .end. */ + t6040_sleep_set_wd2(s, WDT_SUSPEND_SECS); + t6040_wake_sync(); + + if (s->mbox) { + dcache_inval_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + dev_dbg(s->dev, "wake: resumes now %llu\n", READ_ONCE(s->mbox->resumes)); + } + for (i = 0; i < AOP_WAKE_WORDS; i++) + s->wake_status[i] = readl(s->aop_global + AOP_WAKE_STATUS(i)); + s->returned = true; +} + +/* + * Offer "deep" only when the system was booted with allow_entry: without it + * every suspend to RAM is refused, so mem_sleep keeps s2idle as its default. + */ +static int t6040_sleep_valid(suspend_state_t state) +{ + return state == PM_SUSPEND_MEM && READ_ONCE(allow_entry); +} + +/* + * The VDD_CIO rail request (SMC key pmVC) is written in each .begin; its key and value + * belong to the SMC driver (apple_smc_request_pmvc()). + */ + +/* The SMC node is apple,t6041-smc + apple,t8103-smc here (macsmc's match list). */ +static const struct of_device_id t6040_smc_match[] = { + { .compatible = "apple,t8103-smc" }, + { .compatible = "apple,smc" }, + {} +}; + +static void t6040_sleep_pmvc(struct t6040_sleep *s) +{ + struct platform_device *pdev; + struct device_node *np; + struct apple_smc *smc; + int ret = -ENODEV; + + np = of_find_matching_node(NULL, t6040_smc_match); + pdev = np ? of_find_device_by_node(np) : NULL; + of_node_put(np); + smc = pdev ? platform_get_drvdata(pdev) : NULL; + if (smc) + ret = apple_smc_request_pmvc(smc); + if (pdev) + put_device(&pdev->dev); + if (!smc) + dev_warn(s->dev, "pmVC: no SMC to write to: %d\n", ret); +} + +/* + * The refusal-first gate of a suspend to RAM, which t6040_sleep_valid() offers only with + * allow_entry. With a blocker left, print the blocker list and return an error here, before + * any device suspends, so nothing is entered. A dry run proceeds and refuses at .enter. No + * blocker and no dry run: the path that reaches the commit. + */ +static int t6040_sleep_begin(suspend_state_t state) +{ + struct t6040_sleep *s = t6040_sleep; + unsigned int disp = 0; + unsigned int live = t6040_sleep_live_blockers(s, NULL, &disp); + /* + * The display pipes are lowered during device suspend (apple-dcp .suspend), + * so .begin only gates on the non-display live blockers; .enter re-checks the + * full set at the commit point, by which time the display is off. + */ + unsigned int begin_total = live - disp; + unsigned int dry = READ_ONCE(dry_run), secs; + u32 dcs_to; + + /* + * The PM core calls .begin and .end for suspend-to-idle too when there + * are no s2idle_ops. s2idle needs nothing from this controller. + */ + s->mem_begun = false; + if (state != PM_SUSPEND_MEM) + return 0; + s->mem_begun = true; + + t6040_sleep_set_dry(s, 0); + s->returned = false; + s->gp_saved[0] = false; + s->gp_saved[1] = false; + /* + * Decide before anything is written: a refused suspend leaves the AOP islands, the SMC + * rail request and the DCS power words as they were. Only a dry run or a proceeding + * suspend, which both go through device suspend and .enter, needs them set up. + */ + if (begin_total && !dry) { + dev_info(s->dev, "suspend requested: %u live blocker(s) (%u display, cleared during device suspend)\n", + live, disp); + t6040_sleep_log_blockers(s); + dev_info(s->dev, "refusing the suspend; nothing entered\n"); + return -EOPNOTSUPP; + } + + t6040_aop_ps_save(s); + t6040_sleep_pmvc(s); + t6040_dcs_dyn_pg(s); + dcs_to = t6040_dcs_ps_auto_on(s); + if (dcs_to) + dev_warn(s->dev, "DCS power words %#x did not read on before automatic mode\n", + dcs_to); + if (!begin_total && !dry) { + /* + * The non-display blockers are clear and no dry run is asked for: proceed + * to device suspend; .enter holds the real commit until the display is + * also off. The AON WD2 is the net from here. + */ + dev_dbg(s->dev, "entering: %u display pipe(s) on, to be lowered before the commit\n", + disp); + t6040_sleep_power_ops(s); + t6040_sleep_set_wd2(s, WDT_SUSPEND_SECS); + pm_set_suspend_via_firmware(); + return 0; + } + + dev_info(s->dev, "suspend requested: %u live blocker(s) (%u display, cleared during device suspend)\n", + live, disp); + t6040_sleep_log_blockers(s); + + /* A dry run: the refusal case returned above. */ + dry = min(dry, 2U); + t6040_sleep_power_ops(s); + if (!s->power_ops) + dev_warn(s->dev, "dry run: no CPU parking power ops, so no hold at the commit point\n"); + t6040_sleep_set_dry(s, dry); + pm_set_suspend_via_firmware(); + secs = min(READ_ONCE(wd2_secs), WDT_MAX_SECS); + if (secs && s->wdt) + t6040_sleep_set_wd2(s, secs); + else + secs = 0; + dev_info(s->dev, "dry run %u: firmware-driven device sleep short of %s, refused at the commit point, WD2 %u s (0: off)\n", + dry, dry >= 2 ? "the stops only a power cycle undoes" : + "the last steps", secs); + return 0; +} + +/* After the device suspend, in process context. */ +static int t6040_sleep_prepare(void) +{ + t6040_sleep_watch_bound(t6040_sleep); + + return 0; +} + +/* + * A dry run at the commit point, interrupts off and one CPU up: every + * check a real commit makes, the parked CPUs held and let go, at level 2 each + * coprocessor's last check, then the refusal. Never reached: the enclave's + * sleep request, each coprocessor's last stop, the wake masks, the SoC + * power-down. + */ +static int t6040_sleep_dry_enter(struct t6040_sleep *s, unsigned int dry) +{ + u32 p[AOP_WAKE_WORDS]; + unsigned int refused; + int ret, i; + + dev_info(s->dev, "dry run %u at the commit point: %u CPU(s) online\n", dry, + num_online_cpus()); + t6040_sleep_log_s2r_state(s, "dry-run commit point"); + /* + * A wake source pending and left unmasked by the masks a real commit arms would keep the + * AP partition from powering down, and the sleep engine then stalls. Read only. + */ + for (i = 0; i < AOP_WAKE_WORDS; i++) + p[i] = readl(s->aop_global + AOP_WAKE_STATUS(i)) & ~t6040_wake_mask[i]; + dev_info(s->dev, "dry-run commit point: wake status left unmasked by the commit's masks %*ph\n", + (int)sizeof(p), p); + refused = t6040_sleep_refuse(s); + dev_info(s->dev, "a real commit would %s here (%u check(s) failed)\n", + refused ? "refuse" : "go on", refused); + + ret = s->power_ops ? spin_table_hold_parked() : -EOPNOTSUPP; + if (ret) + dev_err(s->dev, "the parked CPUs didn't answer the hold: %d\n", ret); + else + dev_info(s->dev, "%u parked CPU(s) answered the hold\n", + num_present_cpus() - num_online_cpus()); + + if (dry >= 2 && !ret) + dev_info(s->dev, "the coprocessors' last checks: %d\n", + apple_sleep_coprocs_check(s->dev)); + if (!ret) + spin_table_unhold_parked(); + + dev_info(s->dev, "dry run: refused before the enclave's sleep request (%s), the coprocessors' last stops and the SoC power-down\n", + apple_sleep_enclave_registered() ? "left by sep" : "none left"); + + return -EBUSY; +} + +/* + * regmap_read_poll_timeout_atomic() reads ktime_get(), which warns while timekeeping is + * suspended. Counted 1 us steps instead. + */ +static int t6040_regmap_poll_off(struct regmap *map, u32 off, u32 *cur) +{ + unsigned int us; + + for (us = 0; ; us++) { + if (regmap_read(map, off, cur)) + return -EIO; + if (!FIELD_GET(PS_ACTUAL, *cur)) + return 0; + if (us >= PS_OFF_TIMEOUT_US) + return -ETIMEDOUT; + udelay(1); + } +} + +/* A power word behind a regmap (pmgr_disp) or a plain window (soc_east). */ +struct t6040_psw { + struct regmap *map; + void __iomem *io; + u32 off; +}; + +static u32 t6040_psw_read(const struct t6040_psw *w) +{ + u32 v = 0; + + if (w->map) + regmap_read(w->map, w->off, &v); + else + v = readl(w->io + w->off); + return v; +} + +static void t6040_psw_write(const struct t6040_psw *w, u32 v) +{ + if (w->map) + regmap_write(w->map, w->off, v); + else + writel(v, w->io + w->off); +} + +static int t6040_psw_poll_long(const struct t6040_psw *w, u32 mask, u32 want, unsigned int max_us) +{ + unsigned int us; + + for (us = 0; us < max_us; us++) { + if ((t6040_psw_read(w) & mask) == want) + return 0; + udelay(1); + } + return -ETIMEDOUT; +} + +static int t6040_psw_poll(const struct t6040_psw *w, u32 mask, u32 want) +{ + unsigned int us; + + for (us = 0; us < DISP_PS_TIMEOUT_US; us++) { + if ((t6040_psw_read(w) & mask) == want) + return 0; + udelay(1); + } + return -ETIMEDOUT; +} + +/* + * The power-down of a PS word: with + * AUTO_ENABLE set, take it out of automatic mode with the target kept and the flags cleared, and + * wait for ACTUAL 15; then @pre (the SYS hooks); then TARGET 0 with bits 9:8 written 1 and + * bits 10-12, 24-31 kept; then poll bit11 clear and ACTUAL 0. + */ +static int t6040_psw_off(struct t6040_sleep *s, const struct t6040_psw *w, + void (*pre)(struct t6040_sleep *s)) +{ + u32 v = t6040_psw_read(w); + + if (v & PS_AUTO_ENABLE) { + t6040_psw_write(w, v & ~(PS_AUTO_ENABLE | PS_FLAGS)); + t6040_psw_poll(w, PS_ACTUAL, PS_ACTUAL); + v = t6040_psw_read(w); + } + if (pre) + pre(s); + t6040_psw_write(w, (v & ~(PS_FLAGS | PS_TARGET)) | PS_FLAGS); + t6040_psw_poll(w, PS_BUSY, 0); + return t6040_psw_poll(w, PS_ACTUAL, 0); +} + +/* DISP_SYS's hooks before its PS write: the axi2af bridge bit0s, then the retention pair. */ +static void t6040_disp_sys_pre(struct t6040_sleep *s) +{ + void __iomem *bridge = s->axi2af_bridge[AXI2AF_DISP_SYS]; + + if (bridge) { + writel(readl(bridge + AXI2AF_BRIDGE_104) & ~BIT(0), bridge + AXI2AF_BRIDGE_104); + writel(readl(bridge + AXI2AF_BRIDGE_108) & ~BIT(0), bridge + AXI2AF_BRIDGE_108); + } + if (s->disp_pwrgate) { + writel(3, s->disp_pwrgate + DISP_PWRGATE_A); + writel(0, s->disp_pwrgate + DISP_PWRGATE_B); + } +} + +/* dart-dcp's outstanding-transaction counters at 0 before its domain goes. */ +#define DCP_DART_TXN_COUNT GENMASK(20, 16) + +static int t6040_dcp_dart_idle(struct t6040_sleep *s) +{ + static const u16 regs[] = { 0x240, 0x244, 0x250, 0x254, 0x260, 0x270 }; + unsigned int i, us = 0; + + if (!s->dcp_dart) + return 0; + for (i = 0; i < ARRAY_SIZE(regs); i++) + while (readl(s->dcp_dart + regs[i]) & DCP_DART_TXN_COUNT) { + if (++us > DISP_PS_TIMEOUT_US) + return -ETIMEDOUT; + udelay(1); + } + return 0; +} + +/* The internal display's FE and SYS domains, child first, after its sub-blocks. */ +static int t6040_disp_off(struct t6040_sleep *s, const struct t6040_sleep_group *g) +{ + struct t6040_psw cpu = { .map = g->regmap, .off = g->first }; + struct t6040_psw fe = { .io = s->soc_east, .off = PS_DISP_FE }; + struct t6040_psw sys = { .io = s->soc_east, .off = PS_DISP_SYS }; + int ret; + + s->disp_down = true; + /* + * The PMC's bandwidth-request and boost interface enables of the display pipes + * (0x5048ac020 / 0x5048ac028, one bit per pipe; they belong set while a pipe's SYS + * domain is powered). The power loss clears them and nothing sets them + * again: without its boost bit a pipe's backlight data stream never runs. + */ + s->pmc_if_valid = false; + if (s->pmc_bwr) { + s->pmc_if_saved[0] = readl(s->pmc_bwr); + s->pmc_if_saved[1] = readl(s->pmc_bwr + PMC_BOOST); + s->pmc_if_valid = true; + } + /* The PMS DCP voters to 0 first, as T8140's driver ends. */ + if (s->pmc_dcp) { + s->pmc_dcp_saved[0] = readl(s->pmc_dcp); + s->pmc_dcp_saved[1] = readl(s->pmc_dcp + 4); + writel(0, s->pmc_dcp); + writel(0, s->pmc_dcp + 4); + } + /* + * DISP_CPU's word refuses any write from here: rewriting its own value at runtime reset + * the SoC. Its CPU is left in automatic mode and the PMGR gates it once the firmware + * idles in WFI with the run bit kept: wait for ACTUAL 0, after dart-dcp's transactions + * have drained, and go on either way. + */ + if (FIELD_GET(PS_ACTUAL, t6040_psw_read(&cpu))) + t6040_dcp_dart_idle(s); + t6040_psw_poll_long(&cpu, PS_ACTUAL, 0, DISP_CPU_GATE_US); + + ret = t6040_psw_off(s, &fe, NULL); + if (!ret) + ret = t6040_psw_off(s, &sys, t6040_disp_sys_pre); + return ret; +} + +/* + * The raise (the mirror): TARGET 15, poll ACTUAL 15, poll bit11; for DISP_SYS the + * retention pair (A := 0, B := 1) and the axi2af bridge bit0s set; then bits 9:8 cleared and + * AUTO_ENABLE back where it belongs (DISP_CPU, DISP_SYS). + */ +static int t6040_psw_on(struct t6040_sleep *s, const struct t6040_psw *w, bool sys, + bool autoen) +{ + u32 v = t6040_psw_read(w); + int ret; + + t6040_psw_write(w, (v & ~(PS_AUTO_ENABLE | PS_FLAGS)) | PS_TARGET); + ret = t6040_psw_poll(w, PS_ACTUAL, PS_ACTUAL); + t6040_psw_poll(w, PS_BUSY, 0); + if (sys) { + void __iomem *bridge = s->axi2af_bridge[AXI2AF_DISP_SYS]; + + if (s->disp_pwrgate) { + writel(0, s->disp_pwrgate + DISP_PWRGATE_A); + writel(1, s->disp_pwrgate + DISP_PWRGATE_B); + } + if (bridge && !ret) { + writel(readl(bridge + AXI2AF_BRIDGE_104) | BIT(0), + bridge + AXI2AF_BRIDGE_104); + writel(readl(bridge + AXI2AF_BRIDGE_108) | BIT(0), + bridge + AXI2AF_BRIDGE_108); + } + } + v = t6040_psw_read(w); + t6040_psw_write(w, (v & ~PS_AUTO_ENABLE) | PS_FLAGS | (autoen ? PS_AUTO_ENABLE : 0)); + return ret; +} + +/* The internal display back on, parent first, whatever brought .enter back. */ +static void t6040_disp_back(struct t6040_sleep *s) +{ + struct t6040_psw sys = { .io = s->soc_east, .off = PS_DISP_SYS }; + struct t6040_psw fe = { .io = s->soc_east, .off = PS_DISP_FE }; + + if (!s->disp_down) + return; + s->disp_down = false; + /* DISP_CPU's word can't be written from here: automatic mode raises it. */ + if (t6040_psw_on(s, &sys, true, true)) + return; + if (t6040_psw_on(s, &fe, false, false)) + return; + if (s->pmc_dcp) { + writel(s->pmc_dcp_saved[1], s->pmc_dcp + 4); + writel(s->pmc_dcp_saved[0], s->pmc_dcp); + } + if (s->pmc_if_valid) { + u32 bwr = readl(s->pmc_bwr), boost = readl(s->pmc_bwr + PMC_BOOST); + + s->pmc_if_valid = false; + if (bwr != s->pmc_if_saved[0]) + writel(bwr | s->pmc_if_saved[0], s->pmc_bwr); + if (boost != s->pmc_if_saved[1]) + writel(boost | s->pmc_if_saved[1], s->pmc_bwr + PMC_BOOST); + if (bwr != s->pmc_if_saved[0] || boost != s->pmc_if_saved[1]) + dev_dbg(s->dev, "wake: PMC display interfaces BWR %#x boost %#x, put back to %#x %#x\n", + bwr, boost, readl(s->pmc_bwr), readl(s->pmc_bwr + PMC_BOOST)); + } +} + +/* + * The display power words the ADT wants off before a sleep (apple,check-ps-b4-sleep), lowered + * here: after the device suspend, interrupts off, one CPU up, so no driver, interrupt handler + * or DART touches the pipes any more. The DCPs only stop their firmware: lowering the domains + * from their suspend makes the SoC panic. Each group from its last sub-block up to, and not + * including, its pipe CPU, which the PMGR gates by itself: each word as the pmgr-pwrstate + * driver powers a domain off (TARGET 0, the one-shot bits clear), polled to ACTUAL 0. The + * internal display's FE and SYS domains follow. Returns the number of words left on. + */ +static unsigned int t6040_disp_lower(struct t6040_sleep *s) +{ + unsigned int i, left = 0; + + for (i = 0; i < s->ngroups; i++) { + const struct t6040_sleep_group *g = &s->groups[i]; + u32 off = g->last; + + for (;;) { + bool cpu = off == g->first; + u32 val, cur = 0; + + /* An automatically gated CPU still on: wait for the PMGR to gate it. */ + if (cpu && !regmap_read(g->regmap, off, &val) && (val & PS_AUTO_ENABLE) && + FIELD_GET(PS_ACTUAL, val)) + t6040_regmap_poll_off(g->regmap, off, &cur); + if (!cpu && !regmap_read(g->regmap, off, &val) && + t6040_sleep_ps_on(val, cpu)) { + val &= ~(PS_TARGET | PS_AUTO_ENABLE | PS_FLAGS | PS_DEV_DISABLE | + PS_BUSY | PS_UNLATCH); + regmap_write(g->regmap, off, val); + if (t6040_regmap_poll_off(g->regmap, off, &cur)) { + dev_warn(s->dev, "%s +%#x still on: %#x\n", g->name, + off, cur); + left++; + } + } + if (off == g->first) + break; + off -= PS_STRIDE; + } + if (!i && !strcmp(g->name, "disp") && t6040_disp_off(s, g)) + left++; + } + return left; +} + +static int t6040_sleep_enter(suspend_state_t state) +{ + struct t6040_sleep *s = t6040_sleep; + unsigned int dry = s->dry, left; + int ret; + + if (dry) + return t6040_sleep_dry_enter(s, dry); + + left = t6040_disp_lower(s); + if (left) + dev_warn(s->dev, "%u display power word(s) did not go off\n", left); + if (t6040_sleep_live_blockers(s, NULL, NULL)) { + unsigned int on = t6040_sleep_log_groups(s); + + dev_info(s->dev, "at the commit point: %u CPU(s) online, %u display group(s) on; refusing\n", + num_online_cpus(), on); + t6040_disp_back(s); + return -EBUSY; + } + + /* The refusals, the hold and the stops in the generic driver's order. */ + if (t6040_sleep_refuse(s)) { + t6040_disp_back(s); + return -EBUSY; + } + ret = spin_table_hold_parked(); + if (ret) { + t6040_disp_back(s); + return ret; + } + ret = apple_sleep_coprocs_check(s->dev); + if (!ret) + ret = apple_sleep_enclave_stop(s->dev); + if (ret) { + spin_table_unhold_parked(); + t6040_disp_back(s); + return ret; + } + s->pcie_ans_stopped = false; + apple_sleep_coprocs_stop(true, &s->pcie_ans_stopped); + apple_sleep_coprocs_stop(false, &s->pcie_ans_stopped); + t6040_wake_arm(s); + + s->cntkctl = read_sysreg(cntkctl_el1); + pm_set_resume_via_firmware(); + ret = cpu_suspend(0, t6040_sleep_commit); + if (!ret) + t6040_sleep_return(s); + + if (s->returned) + t6040_wake_sync(); + t6040_wake_restore(s); + if (s->returned) { + t6040_wake_sync(); + t6040_sio_ps_restore(s); + } + t6040_disp_back(s); + if (s->returned) { + t6040_wake_sync(); + if (s->pcie_ans_on_ret) + dev_warn(s->dev, "wake: the storage PCIe raise failed after step %u: %d\n", + s->apcie_wake_step, s->pcie_ans_on_ret); + if (s->gp_clk_timeouts) + dev_warn(s->dev, "wake: %u GP PCIe clock or ready poll(s) timed out\n", + s->gp_clk_timeouts); + if (s->aop_ps_timeouts) + dev_warn(s->dev, "wake: %u AOP island power word(s) did not come on\n", + s->aop_ps_timeouts); + } + return ret; +} + +static void t6040_sleep_end(void) +{ + struct t6040_sleep *s = t6040_sleep; + unsigned int dry = s->dry; + + if (!s->mem_begun) + return; + s->mem_begun = false; + t6040_sleep_clear_wd2(s); + if (dry) + dev_info(s->dev, "dry run %u over, WD2 off\n", dry); + t6040_sleep_set_dry(s, 0); +} + +static const struct platform_suspend_ops t6040_sleep_ops = { + .valid = t6040_sleep_valid, + .begin = t6040_sleep_begin, + .prepare = t6040_sleep_prepare, + .enter = t6040_sleep_enter, + .end = t6040_sleep_end, +}; + +static int t6040_sleep_parse_groups(struct t6040_sleep *s) +{ + struct device_node *np = s->dev->of_node; + const char *prop = "apple,check-ps-b4-sleep"; + int n, i; + + /* Each group: the PMGR syscon, the CPU's offset and the last sub-block's. */ + n = of_property_count_u32_elems(np, prop); + if (n <= 0 || n % 3) + return dev_err_probe(s->dev, -EINVAL, "no valid %s\n", prop); + n /= 3; + if (of_property_count_strings(np, "apple,check-ps-b4-sleep-names") != n) + return dev_err_probe(s->dev, -EINVAL, "one name per %s group\n", prop); + + s->groups = devm_kcalloc(s->dev, n, sizeof(*s->groups), GFP_KERNEL); + if (!s->groups) + return -ENOMEM; + + for (i = 0; i < n; i++) { + struct t6040_sleep_group *g = &s->groups[i]; + struct of_phandle_args args; + int ret; + + ret = of_parse_phandle_with_fixed_args(np, prop, 2, i, &args); + if (ret) + return dev_err_probe(s->dev, ret, "%s[%d]\n", prop, i); + g->regmap = syscon_node_to_regmap(args.np); + of_node_put(args.np); + if (IS_ERR(g->regmap)) + return dev_err_probe(s->dev, PTR_ERR(g->regmap), "%s[%d] regmap\n", + prop, i); + g->first = args.args[0]; + g->last = args.args[1]; + if (g->first > g->last || (g->last - g->first) % PS_STRIDE || + (g->last - g->first) / PS_STRIDE >= PS_MAX_PER_GROUP) + return dev_err_probe(s->dev, -EINVAL, "%s[%d]: bad range %#x..%#x\n", + prop, i, g->first, g->last); + of_property_read_string_index(np, "apple,check-ps-b4-sleep-names", i, &g->name); + } + s->ngroups = n; + + return 0; +} + +/* + * A watched device's own power word: its first power domain, a PMGR power-state + * node whose reg is the offset in its parent's syscon. Optional. + */ +static void t6040_sleep_watch_init(struct t6040_sleep *s, unsigned int i, + const char *compatible, const char *name) +{ + struct t6040_sleep_watch *w = &s->watch[i]; + struct device_node *np, *pd, *pmgr; + struct regmap *regmap; + + w->compatible = compatible; + np = of_find_compatible_node(NULL, NULL, compatible); + if (!np) + return; + pd = of_parse_phandle(np, "power-domains", 0); + of_node_put(np); + if (!pd) + return; + pmgr = of_get_parent(pd); + if (pmgr && !of_property_read_u32(pd, "reg", &w->ps_offset)) { + regmap = syscon_node_to_regmap(pmgr); + if (!IS_ERR(regmap)) { + w->ps_regmap = regmap; + w->has_ps = true; + } + } + of_node_put(pmgr); + of_node_put(pd); + if (!w->has_ps) + dev_warn(s->dev, "%s: no power word to watch\n", name); +} + +/* + * A named register block, shared with the owning driver: mapped, not claimed. NULL when it is + * missing from the node or smaller than @min, the extent the driver accesses. + */ +static void __iomem *t6040_sleep_map_min(struct device *dev, const char *name, size_t min) +{ + struct device_node *np = dev->of_node; + struct resource res; + int index; + + index = of_property_match_string(np, "reg-names", name); + if (index < 0 || of_address_to_resource(np, index, &res) || resource_size(&res) < min) + return NULL; + + return t6040_ioremap_np(dev, res.start, resource_size(&res)); +} + +static void __iomem *t6040_sleep_map(struct device *dev, const char *name) +{ + return t6040_sleep_map_min(dev, name, 0); +} + +/* + * ANS's storage-apcie windows + PLL/lane tunables, from the apple,apcie node + * the boot object publishes from the ADT. Our static tree has none + * of these, so until aura fills the node @valid stays false and a real sleep + * with ANS stopped refuses (t6040_sleep_refuse). The reg-names and tunable + * property names match the T8140's apple,apcie shape exactly, so the same + * loader fills both SoCs. The GP PCIe host (pcie@1cb0000000, Wi-Fi and SD) is a + * different node, mapped by t6040_sleep_map_gp(). + */ +static void t6040_sleep_map_apcie(struct t6040_sleep *s) +{ + struct t6040_apcie_st *a = &s->apcie_st; + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,apcie", 0); + + if (!np) + return; + a->axi = apple_sleep_apcie_block(s->dev, np, "axi2af", "apple,tunable-axi2af", + &a->axi_tunable); + a->cio3pll = apple_sleep_apcie_block(s->dev, np, "cio3pll", "apple,tunable-cio3pll", + &a->cio3pll_tunable); + a->clkgen = apple_sleep_apcie_block(s->dev, np, "clkgen", "apple,tunable-clkgen", + &a->clkgen_tunable); + a->phy = apple_sleep_apcie_block(s->dev, np, "phy-block", "apple,tunable-phy", + &a->phy_tunable); + a->rc = apple_sleep_apcie_block(s->dev, np, "rc", NULL, NULL); + a->phy_ip = apple_sleep_apcie_block(s->dev, np, "phy-ip", "apple,tunable-phy-ip-pll", + &a->phy_ip_pll); + a->phy_ip_lanes = apple_sleep_apcie_tunable(s->dev, np, "phy-ip", + "apple,tunable-phy-ip-auspma"); + of_node_put(np); + + a->valid = a->axi && a->cio3pll && a->clkgen && a->phy && a->rc && a->phy_ip && + a->phy_ip_lanes; + if (!a->valid) + dev_info(s->dev, + "no storage-apcie windows/tunables (apple,apcie): a real sleep with ANS stopped refuses\n"); +} + +/* Maps the mailbox at @pa if it is aura's, or returns NULL. */ +static struct aura_s2r_mailbox *t6040_sleep_mailbox_at(struct t6040_sleep *s, u64 pa) +{ + struct aura_s2r_mailbox *mbox; + + mbox = devm_memremap(s->dev, pa, sizeof(*mbox), MEMREMAP_WB); + if (IS_ERR(mbox)) + return NULL; + if (mbox->magic != AURA_S2R_MAGIC) { + dev_warn(s->dev, "s2r-mailbox magic %#llx is not AURAS2R1\n", mbox->magic); + devm_memunmap(s->dev, mbox); + return NULL; + } + return mbox; +} + +/* The boot object's mailbox, if it published one. Optional (a blocker if absent). */ +static void t6040_sleep_map_mailbox(struct t6040_sleep *s) +{ + struct device_node *chosen = of_find_node_by_path("/chosen"); + u64 pa; + int ret; + + if (!chosen) + return; + ret = of_property_read_u64(chosen, "aurora,s2r-mailbox", &pa); + of_node_put(chosen); + if (ret) + return; + + s->mbox = t6040_sleep_mailbox_at(s, pa); + if (s->mbox) + s->mbox_pa = pa; +} + +#ifdef CONFIG_HIBERNATION +/* + * Restored from a hibernation image, this kernel runs under the aura of the boot that resumed + * it. That aura moved with the boot object, and so did its mailbox: the one mapped at probe was + * the hibernating boot's, and the boot core would come back from a sleep through this boot's. + * Take this boot's address from the device tree aura keeps at its fixed slot. Without it, drop + * the mailbox, so that a suspend refuses for it instead of arming memory that is no longer + * aura's. + */ +static int t6040_sleep_restore(struct device *dev) +{ + struct t6040_sleep *s = dev_get_drvdata(dev); + struct aura_s2r_mailbox *mbox = NULL; + u64 pa = 0; + int ret; + + ret = aura_hibernate_boot_chosen_u64("aurora,s2r-mailbox", &pa); + if (!ret && s->mbox && pa == s->mbox_pa) + return 0; + if (!ret) + mbox = t6040_sleep_mailbox_at(s, pa); + if (s->mbox) + devm_memunmap(dev, s->mbox); + if (mbox) + dev_info(dev, "restored: s2r-mailbox %#llx, was %#llx on the boot that hibernated\n", + pa, s->mbox_pa); + else + dev_warn(dev, "restored: this boot's s2r-mailbox is unknown (%d, %#llx): dropped\n", + ret, pa); + s->mbox = mbox; + s->mbox_pa = mbox ? pa : 0; + return 0; +} +#endif + +/* The GP PCIe host's port and PHY windows and its ports' PERST# pins, for the commit. */ +static void t6040_sleep_map_gp(struct t6040_sleep *s) +{ + static const char *const ports[] = { "port0", "port1" }; + static const char *const phys[] = { "phy0", "phy1" }; + struct device_node *host, *child; + struct device_node *pins = NULL; + struct resource res; + int p, idx; + + s->gp_perst[0] = -1; + s->gp_perst[1] = -1; + host = of_find_compatible_node(NULL, NULL, "apple,t6041-pcie"); + if (!host) + return; + if (!of_device_is_available(host)) + goto out; + for (p = 0; p < ARRAY_SIZE(s->gp_port); p++) { + idx = of_property_match_string(host, "reg-names", ports[p]); + if (idx >= 0 && !of_address_to_resource(host, idx, &res)) + s->gp_port[p] = t6040_ioremap_np(s->dev, res.start, resource_size(&res)); + idx = of_property_match_string(host, "reg-names", phys[p]); + if (idx >= 0 && !of_address_to_resource(host, idx, &res)) + s->gp_phy[p] = t6040_ioremap_np(s->dev, res.start, resource_size(&res)); + } + for_each_available_child_of_node(host, child) { + struct of_phandle_args args; + u32 reg[1]; + + if (of_property_read_u32_array(child, "reg", reg, 1)) + continue; + p = (reg[0] >> 11) & 0x1f; /* the device number of pci@N,0 */ + if (p > 1 || of_parse_phandle_with_args(child, "reset-gpios", "#gpio-cells", 0, + &args)) + continue; + if (args.args_count >= 2 && (!pins || pins == args.np)) { + if (!pins) + pins = of_node_get(args.np); + s->gp_perst[p] = args.args[0]; + s->gp_perst_low[p] = args.args[1] & 1; /* GPIO_ACTIVE_LOW */ + } + of_node_put(args.np); + } + if (pins) { + if (!of_address_to_resource(pins, 0, &res)) + s->gp_pins = t6040_ioremap_np(s->dev, res.start, resource_size(&res)); + of_node_put(pins); + } + dev_dbg(s->dev, "GP PCIe: ports %d/%d phys %d/%d, PERST# pins %d/%d\n", + !!s->gp_port[0], !!s->gp_port[1], !!s->gp_phy[0], !!s->gp_phy[1], + s->gp_perst[0], s->gp_perst[1]); +out: + of_node_put(host); +} + +static int t6040_sleep_probe(struct platform_device *pdev) +{ + struct t6040_sleep *s; + struct dentry *dir; + int ret, i; + + s = devm_kzalloc(&pdev->dev, sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->dev = &pdev->dev; + + ret = t6040_sleep_parse_groups(s); + if (ret) + return ret; + + /* + * The commit's and the wake's register blocks are mandatory: without them a suspend + * could not be entered or undone, and the node always lists them. The axi2af fabric + * bridges are fixed PAs, not DT. + */ + BUILD_BUG_ON(ARRAY_SIZE(t6040_axi2af) != T6040_AXI2AF_DOMAINS); + BUILD_BUG_ON(ARRAY_SIZE(t6040_tvm) != ARRAY_SIZE(((struct t6040_sleep *)0)->tvm_ctl)); + BUILD_BUG_ON(ARRAY_SIZE(t6040_acc_apsc_pa) != + ARRAY_SIZE(((struct t6040_sleep *)0)->acc_apsc)); + s->cpu_start = t6040_sleep_map(&pdev->dev, "cpu-start"); + s->wdt = t6040_sleep_map(&pdev->dev, "wdt"); + s->cluster_ctl = t6040_sleep_map(&pdev->dev, "cluster"); + s->pwrgate = t6040_sleep_map(&pdev->dev, "pwrgate"); + s->clkgen = t6040_sleep_map(&pdev->dev, "clkgen"); + s->spmi0 = t6040_sleep_map(&pdev->dev, "spmi"); + s->spmi_a0 = t6040_sleep_map(&pdev->dev, "spmi-a0"); + s->mastermini = t6040_sleep_map(&pdev->dev, "mastermini"); + s->soc_east = t6040_sleep_map_min(&pdev->dev, "soc-east", SOC_EAST_SIZE); + s->soc_ps = t6040_sleep_map_min(&pdev->dev, "soc-ps", SOC_PS_SIZE); + s->aop_global = t6040_sleep_map(&pdev->dev, "wake"); + if (!s->cpu_start || !s->wdt || !s->cluster_ctl || !s->pwrgate || !s->clkgen || + !s->spmi0 || !s->spmi_a0 || !s->mastermini || !s->soc_east || !s->soc_ps || + !s->aop_global) + return dev_err_probe(&pdev->dev, -ENODEV, + "a register block is missing or not mapped (cpu-start %d wdt %d cluster %d pwrgate %d clkgen %d spmi %d spmi-a0 %d mastermini %d soc-east %d soc-ps %d wake %d)\n", + !!s->cpu_start, !!s->wdt, !!s->cluster_ctl, !!s->pwrgate, + !!s->clkgen, !!s->spmi0, !!s->spmi_a0, !!s->mastermini, + !!s->soc_east, !!s->soc_ps, !!s->aop_global); + { + struct resource wres; + int widx = of_property_match_string(pdev->dev.of_node, "reg-names", "wdt"); + + s->wdt_stub_ok = widx >= 0 && + !of_address_to_resource(pdev->dev.of_node, widx, &wres) && + wres.start == T6040_WDT_PA; + if (!s->wdt_stub_ok) + dev_warn(&pdev->dev, + "the wdt block is not the AON watchdog: no WD2 restart at the wake\n"); + } + s->aop_ps = t6040_ioremap_np(&pdev->dev, T6040_AOP_PS_PA, 0x160); + for (i = 0; i < ARRAY_SIZE(s->dcs); i++) + s->dcs[i] = t6040_ioremap_np(&pdev->dev, 0x220200000ULL + + (i / 4) * 0x2000000ULL + (i % 4) * 0x200000ULL, + 0x2000); + for (i = 0; i < ARRAY_SIZE(s->atc_cfg); i++) + s->atc_cfg[i] = t6040_ioremap_np(&pdev->dev, + 0x383200000ULL + i * 0x8000000ULL, 0x100); + for (i = 0; i < T6040_AXI2AF_DOMAINS; i++) + s->axi2af_bridge[i] = t6040_ioremap_np(&pdev->dev, + t6040_axi2af[i].bridge_pa, 0x200); + t6040_sleep_map_apcie(s); + t6040_sleep_map_mailbox(s); + t6040_sleep_watch_init(s, T6040_WATCH_SEP, "apple,sep", "sep"); + t6040_sleep_watch_init(s, T6040_WATCH_ANS, "apple,t6041-nvme-ans2", "ans"); + /* Fixed PAs: dart-dcp's counters and the internal display's retention pair. */ + s->dcp_dart = t6040_ioremap_np(&pdev->dev, T6040_DCP_DART_PA, 0x1000); + s->disp_pwrgate = t6040_ioremap_np(&pdev->dev, T6040_DISP_PWRGATE_PA, 0x100); + s->pmc_dcp = t6040_ioremap_np(&pdev->dev, T6040_PMC_DCP_PA, 12); + s->pmc_bwr = t6040_ioremap_np(&pdev->dev, T6040_PMC_BWR_PA, 0x10); + s->ka_soc = t6040_ioremap_np(&pdev->dev, T6040_KA_SOC_PA, KA_SIZE); + s->ka_east = t6040_ioremap_np(&pdev->dev, T6040_KA_EAST_PA, KA_SIZE); + s->smc_wdt_deadline = t6040_ioremap_np(&pdev->dev, T6040_SMC_WDT_DEADLINE_PA, 4); + s->agx_gate = t6040_ioremap_np(&pdev->dev, T6040_AGX_GATE_PA, 4); + s->s2r_lo_count = t6040_ioremap_np(&pdev->dev, T6040_S2R_LO_COUNT_PA, 4); + for (i = 0; i < ARRAY_SIZE(t6040_acc_apsc_pa); i++) + s->acc_apsc[i] = t6040_ioremap_np(&pdev->dev, t6040_acc_apsc_pa[i], 8); + for (i = 0; i < ARRAY_SIZE(t6040_tvm); i++) { + s->tvm_ctl[i] = t6040_ioremap_np(&pdev->dev, t6040_tvm[i].ctl, 4); + s->tvm_status[i] = t6040_ioremap_np(&pdev->dev, t6040_tvm[i].status, 4); + } + s->afr_ps = t6040_ioremap_np(&pdev->dev, T6040_AFR_PS_PA, 4); + t6040_sleep_map_gp(s); + t6040_ed_map(&pdev->dev); + + platform_set_drvdata(pdev, s); + t6040_sleep = s; + apple_sleep_registry_ready(); + + dir = debugfs_create_dir("t6040-sleep", NULL); + debugfs_create_file("preflight", 0400, dir, s, &t6040_sleep_preflight_fops); + + /* + * The CPU power ops act only when a commit stops the parked secondaries, and a dry run + * needs them for the hold. Both are suspends to RAM, offered only with allow_entry, so + * only then are they registered (.begin tries again if this failed). Without it kexec + * keeps refusing parking CPUs as before. Parking itself needs no power ops. + */ + if (allow_entry) + t6040_sleep_power_ops(s); + + suspend_set_ops(&t6040_sleep_ops); + dev_info(&pdev->dev, + "refusal-first: %u display groups, storage-apcie %s, s2r-mailbox %s, parking ops %s, dry run %u\n", + s->ngroups, + s->apcie_st.valid ? "mapped" : "absent", + s->mbox ? "present" : "absent", + s->power_ops ? "registered" : "not registered", READ_ONCE(dry_run)); + t6040_sleep_log_s2r_state(s, "probe"); + + return 0; +} + +const struct apple_sleep_part t6040_sleep_part = { + .probe = t6040_sleep_probe, +#ifdef CONFIG_HIBERNATION + .restore = t6040_sleep_restore, +#endif + /* + * Every enclave is taken, in a dry run as in a real sleep. This sleep was tested with + * neither the lid's state at the wake nor USB wake ports. + */ + .registry_flags = APPLE_SLEEP_REG_ENCLAVE_ANY, +}; diff --git a/drivers/soc/apple/sleep-test.c b/drivers/soc/apple/sleep-test.c index d16544746d909..267f11297864b 100644 --- a/drivers/soc/apple/sleep-test.c +++ b/drivers/soc/apple/sleep-test.c @@ -9,6 +9,9 @@ static const char *const apple_sleep_test_tested[] = { "apple,t8140-sleep", "apple,t8132-sleep", +#if APPLE_SLEEP_T6040 + "apple,t6040-sleep", +#endif }; static bool apple_sleep_test_has(const struct apple_sleep_words *all, u32 offset) @@ -77,6 +80,9 @@ static void apple_sleep_test_words(struct kunit *test) }; int i; + /* A SoC with its own sequences has no lists. */ + if (soc->part) + continue; apple_sleep_test_full(test, c, "ps_all", all); apple_sleep_test_full(test, c, "ps_aon_all", aon); for (i = 0; i < ARRAY_SIZE(named); i++) @@ -114,6 +120,9 @@ static void apple_sleep_test_ops(struct kunit *test) const struct apple_sleep_soc *soc = id->data; const struct apple_sleep_ops *ops = soc->ops; + /* A SoC with a part of its own (T6040) brings its sequences there. */ + if (soc->part) + continue; KUNIT_ASSERT_NOT_NULL_MSG(test, ops, "%s has no operations", id->compatible); KUNIT_EXPECT_NOT_NULL(test, ops->deep_wfi_arm); KUNIT_EXPECT_NOT_NULL(test, ops->lower_io); @@ -134,6 +143,8 @@ static void apple_sleep_test_books(struct kunit *test) for (id = apple_sleep_of_match; id->compatible[0]; id++) { const struct apple_sleep_soc *soc = id->data; + if (soc->part) + continue; KUNIT_EXPECT_LE(test, soc->nstray, BITS_PER_LONG); KUNIT_EXPECT_LE(test, soc->ngate_coprocs, ARRAY_SIZE(((struct apple_sleep *)0)->gate)); @@ -148,6 +159,8 @@ static void apple_sleep_test_wake_names(struct kunit *test) const struct apple_sleep_soc *soc = id->data; int i, j; + if (soc->part) + continue; KUNIT_EXPECT_NE(test, soc->wake_last_word, 0); for (i = 0; i < soc->nwake_names; i++) { u32 n = soc->wake_names[i].id; @@ -171,6 +184,8 @@ static void apple_sleep_test_layout(struct kunit *test) for (id = apple_sleep_of_match; id->compatible[0]; id++) { const struct apple_sleep_soc *soc = id->data; + if (soc->part) + continue; KUNIT_EXPECT_GE(test, soc->ncluster_cpu, 1); KUNIT_EXPECT_LE(test, soc->ncluster_cpu, SLEEP_CPU_CLUSTERS_MAX); KUNIT_EXPECT_GT(test, soc->cpu_start_stride, 0); diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index 202c07c098947..15b844d046b42 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -30,6 +30,10 @@ * T8132 takes the same path with its own tables and operations, t8132_sleep_ops, * measured the same way from a macOS sleep under aura's hypervisor. Its blocks are * T8140's at the same offsets from a SoC base 0x80000000 higher unless noted. + * + * A SoC whose sequences differ in more than its lists brings them as a part + * (sleep-internal.h), and shares the registry of coprocessors with the rest: + * today T6040, in sleep-t6040.c. */ #include @@ -59,6 +63,7 @@ #include #include #include +#include #include #include #include @@ -78,34 +83,11 @@ #include #include -/* Must match aura's struct aura_s2r_mailbox. */ -#define AURA_S2R_MAGIC 0x3152325341525541ULL /* "AURAS2R1" */ - -struct aura_s2r_mailbox { - u64 magic; /* aura, before the kernel starts */ - u64 boot_mpidr; /* aura: the only core that may take resume_pa */ - u64 resume_pa; /* kernel, armed before a commit; aura clears it */ - u64 resumes; /* aura: resume_pa taken this many times */ -}; - -/* PS word */ -#define PS_AUTO_ENABLE BIT(28) -#define PS_UNLATCH BIT(12) -#define PS_DEV_DISABLE BIT(10) -#define PS_FLAGS GENMASK(9, 8) -#define PS_ACTUAL GENMASK(7, 4) -#define PS_TARGET GENMASK(3, 0) +#include "sleep-internal.h" /* Clock protection, per slice: its control word, and the bit that guards the slice's range. */ #define PS_PROT_SLICE(n) (0x100 + 0x40 * (n)) #define PS_PROT_SLICE_ON BIT(4) -#define PS_OFF_TIMEOUT_US 5000 -#define PS_BUSY_SETTLE_US 1000 - -/* CPU start block, as aura's smp_start_cpu() uses it. A core's bit is the SoC's. */ -#define CPU_START_STOP 0x0 -#define CPU_START_SYS 0x4 -#define CPU_START_CLUSTER(n) (0x8 + (n) * 4) /* Power-gate page: reads back this value already. Purpose unknown. */ #define PWRGATE_ON 0x80000000 @@ -121,41 +103,12 @@ struct aura_s2r_mailbox { #define CLUSTER_TIMEOUT_US 5000 -/* Watchdog block, ADT /arm-io/wdt reg[0]. WD2 is the sleep safety net. */ -#define WDT_WD0_COUNT 0x00 -#define WDT_WD0_CTL 0x0c -#define WDT_WD1_COUNT 0x10 -#define WDT_WD2_COUNT 0x20 -#define WDT_WD2_ALARM 0x28 -#define WDT_WD2_CTL 0x2c -#define WDT_TICKS_PER_S 24000000 -/* The sleep's own safety net: ADT /arm-io/wdt simple-reconfig-wdog-icc-time. */ -#define WDT_SLEEP_SECS 5 -/* Device suspend and resume can take a while, and nothing else recovers a hang in them. */ -#define WDT_SUSPEND_SECS 120 - #define AON_AP_DEADLINE 0x08 /* Wake sources: a mask, so 0 enables. Status words are sticky. Where and how many: the SoC. */ #define WAKE_WORDS 9 #define WAKE_SOURCES_MAX 32 -/* The AP queue of the PMU's SPMI controller, and its response vector's enable and ack */ -#define SPMI_STATUS 0x00 -#define SPMI_STATUS_RX_EMPTY BIT(24) -#define SPMI_CMD 0x04 -#define SPMI_REPLY 0x08 -#define SPMI_NOTIFY_ENABLE 0x40 -#define SPMI_NOTIFY_REPLY BIT(0) -#define SPMI_NOTIFY_ACK 0x80 -#define SPMI_CMD_ACTIVE BIT(15) -#define SPMI_OP_EXT_WRITE(n) ((n) - 1) -#define SPMI_OP_EXT_READ(n) (0x20 | ((n) - 1)) -#define SPMI_OP_ZERO_WRITE 0x80 -#define SPMI_OP_SLEEP 0x11 -#define SPMI_OP_WAKEUP 0x13 -#define SPMI_POLL_US 20000 - /* A port controller on SPMI: registers go through a page select, then a data window. */ #define HPM_REG_SELECT 0x00 #define HPM_REG_DATA 0x20 @@ -176,9 +129,6 @@ struct aura_s2r_mailbox { #define HPM_CONTRACT_MS 1500 #define HPM_RESET_TRIES 3 -/* A one-byte register write on the sleep's SPMI bus, to the PMU's USID */ -#define SPMI_CMD_WRITE (BIT(15) | 0x30) -#define SPMI_CMD_USID GENMASK(11, 8) /* T8132's power words where they aren't T8140's */ #define T8132_PS_ATC0_COMMON 0x378 @@ -232,99 +182,8 @@ struct aura_s2r_mailbox { #define AMCC_WORD0_DSID_AGE_OUT BIT(25) #define AMCC_WORD1_UNEXP_RT_HIT BIT(8) -/* A coprocessor's ASC */ -#define ASC_CPU_CONTROL 0x44 -#define ASC_CPU_CONTROL_RUN BIT(4) -#define ASC_CPU_STATUS 0x48 - -/* The part of the PCIe block that ANS shares, whether or not its host is bound. */ -#define APCIE_AXI_EN0 0x104 -#define APCIE_AXI_EN1 0x108 -#define APCIE_CLK_CTRL 0x0 -#define APCIE_CLK_BUSY BIT(31) -#define APCIE_CLK_RUN BIT(5) -#define APCIE_CLK_OFF BIT(1) -#define APCIE_CLK_OFF2 BIT(2) -#define APCIE_CLK_START 0x4 -#define APCIE_CLK_START_GO BIT(31) -#define APCIE_PHY_COMMON 0x4000 -#define APCIE_PHY_CTRL 0x8000 -#define APCIE_PHY_CTRL_CLK0REQ BIT(0) -#define APCIE_PHY_CTRL_CLK1REQ BIT(1) -#define APCIE_PHY_CTRL_CLK0ACK BIT(2) -#define APCIE_PHY_CTRL_CLK1ACK BIT(3) -#define APCIE_PHY_CTRL_RESET BIT(4) -#define APCIE_PHY_CTRL_UP BIT(27) -#define APCIE_PHY_RUN 0x8004 -#define APCIE_PHY_RUN_EN BIT(0) -#define APCIE_PHY_RUN_EN2 BIT(4) -#define APCIE_PHY_READY 0x8008 -#define APCIE_PHY_IP_PLL_CTL 0x90 -#define APCIE_PHY_IP_PLL_HOLD BIT(16) -#define APCIE_PORT_INTSTAT 0x100 -#define APCIE_PORT_INTMSK 0x104 -#define APCIE_PORT_INT_LINK GENMASK(7, 4) -#define APCIE_PORT_L2 0x88 -#define APCIE_PORT_L2_ENTER 0x11 -#define APCIE_PORT_L2_BUSY BIT(0) -#define APCIE_PORT_PERST_GUARD 0x13c -#define APCIE_PORT_LINKSTS 0x208 -#define APCIE_PORT_LINKSTS_UP BIT(0) -#define APCIE_PORT_APPCLK 0x800 -#define APCIE_PORT_APPCLK_EN BIT(0) -#define APCIE_PORT_APPCLK_CGDIS BIT(8) -#define APCIE_PORT_PERST 0x82c -#define APCIE_PORT_PHY_LINK BIT(31) -#define APCIE_PORT_PHY_EN0 BIT(9) -#define APCIE_PORT_PHY_EN1 BIT(10) -#define APCIE_ENGINE_CTRL 0x0 -#define APCIE_ENGINE_IRQ_MASK 0x8 -#define APCIE_ENGINE_STOP BIT(0) -#define APCIE_FABRIC_STOP 0x4 -#define APCIE_PORT_STOP_US 5000 -#define APCIE_RC_ZERO 0x04 -#define APCIE_RC_READY_START 0x50 -#define APCIE_RC_READY_CFG 0x54 -#define APCIE_RC_READY_STAT 0x58 -#define APCIE_POLL_US 50000 - -/* The coprocessors the last mile stops. Each driver idles its own first. */ -#define SLEEP_COPROCS 4 - -static struct apple_sleep_coproc { - struct device *dev; - void __iomem *asc; - bool pcie_ans; - int (*last_check)(void *data); - int (*last_stop)(void *data); - void *data; - bool ready; - bool stopped; - u32 status; - int stop_ret; - /* No ASC: its last stop takes it whole, power word @ps included, before the others. */ - bool first; - u32 ps; -} apple_sleep_coprocs[SLEEP_COPROCS]; -static DEFINE_MUTEX(apple_sleep_coprocs_lock); - -/* Irreversible enclave stop; nonzero is legal only if SLEEP was not sent. */ -static struct { - struct device *dev; - int (*stop)(void *data); - void *data; -} apple_sleep_enclave; - -/* - * Written with bits 0 and 63 by every core right before its last WFI, measured - * from macOS. This is not CYC_OVRD, which is undefined on this SoC. - */ -#define SYS_APL_DEEP_SLEEP sys_reg(3, 5, 15, 6, 2) -#define DEEP_SLEEP_ENABLE (BIT_ULL(0) | BIT_ULL(63)) - -#define SYS_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) -#define IPI_SR_PENDING BIT(0) -#define SYS_APL_SIQ_CFG_EL1 sys_reg(3, 4, 15, 10, 4) +/* The coprocessor and enclave registry is shared with the other SoCs' controllers. */ +#define SLEEP_COPROCS APPLE_SLEEP_COPROCS #define SLEEP_DISPLAYS_MAX 3 @@ -527,6 +386,8 @@ struct apple_sleep_soc { /* The PMC whose cluster vote for the encoders needs the complex up, and its bit */ const char *ave_pmc; u32 ave_pmc_cluster; + /* The SoC's own sequences, in place of the lists above, or NULL */ + const struct apple_sleep_part *part; }; struct apple_sleep { @@ -689,17 +550,6 @@ struct apple_sleep { static struct apple_sleep *apple_sleep; -static struct apple_sleep_coproc *apple_sleep_coproc_find(struct device *dev) -{ - int i; - - for (i = 0; i < SLEEP_COPROCS; i++) - if (apple_sleep_coprocs[i].dev == dev) - return &apple_sleep_coprocs[i]; - - return NULL; -} - /* A registered coprocessor its driver has readied, whose run bit the last step clears. */ static bool apple_sleep_coproc_ready_node(const struct device_node *np) { @@ -720,20 +570,6 @@ void apple_sleep_usb_set_asleep(bool asleep) } EXPORT_SYMBOL_GPL(apple_sleep_usb_set_asleep); -static unsigned long sleep_usb_wake_ports; - -void apple_sleep_usb_set_wake_ports(unsigned long ports) -{ - WRITE_ONCE(sleep_usb_wake_ports, ports); -} -EXPORT_SYMBOL_GPL(apple_sleep_usb_set_wake_ports); - -bool apple_sleep_usb_port_can_wake(unsigned int port) -{ - return port < BITS_PER_LONG && (READ_ONCE(sleep_usb_wake_ports) & BIT(port)); -} -EXPORT_SYMBOL_GPL(apple_sleep_usb_port_can_wake); - /* The USB host's vote, kept until the word is mapped. Only bit 0 is touched, as on macOS. */ static DEFINE_SPINLOCK(sleep_slp_ddr_lock); static bool sleep_slp_ddr_usb; @@ -897,200 +733,23 @@ void apple_sleep_restore_pins(void) } EXPORT_SYMBOL_GPL(apple_sleep_restore_pins); -/* Below EL2 a monitor owns the last mile, and this code doesn't run it. */ -int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, - int (*last_stop)(void *data), void *data) -{ - struct apple_sleep_coproc *c; - - if (!is_kernel_in_hyp_mode()) - return -ENODEV; - - guard(mutex)(&apple_sleep_coprocs_lock); - if (apple_sleep_coproc_find(dev)) - return -EEXIST; - c = apple_sleep_coproc_find(NULL); - if (!c) - return -ENOSPC; - c->asc = asc; - c->pcie_ans = pcie_ans; - c->last_check = NULL; - c->last_stop = last_stop; - c->data = data; - c->ready = false; - c->stopped = false; - c->first = false; - c->ps = 0; - c->dev = dev; - - return 0; -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_register); - -int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), - void *data) -{ - struct apple_sleep_coproc *c; - struct device_node *np; - u32 ps = 0; - int ret; - - if (!last_stop) - return -EINVAL; - np = of_parse_phandle(dev->of_node, "power-domains", 0); - if (np) { - of_property_read_u32(np, "reg", &ps); - of_node_put(np); - } - ret = apple_sleep_coproc_register(dev, NULL, false, last_stop, data); - if (ret) - return ret; - - guard(mutex)(&apple_sleep_coprocs_lock); - c = apple_sleep_coproc_find(dev); - if (c) { - c->first = true; - c->ps = ps; - } - - return 0; -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_register_first); - -int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)) -{ - struct apple_sleep_coproc *c; - - guard(mutex)(&apple_sleep_coprocs_lock); - c = apple_sleep_coproc_find(dev); - if (!c) - return -ENOENT; - c->last_check = last_check; - - return 0; -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_last_check); - -int apple_sleep_enclave_register(struct device *dev, int (*stop)(void *data), void *data) -{ - if (!is_kernel_in_hyp_mode()) - return -ENODEV; - - if (!stop) - return -EINVAL; - if (!apple_sleep_commit_stops_coprocs()) - return -EOPNOTSUPP; - - guard(mutex)(&apple_sleep_coprocs_lock); - if (apple_sleep_enclave.dev) - return -EBUSY; - apple_sleep_enclave.stop = stop; - apple_sleep_enclave.data = data; - apple_sleep_enclave.dev = dev; - - return 0; -} -EXPORT_SYMBOL_GPL(apple_sleep_enclave_register); - -void apple_sleep_enclave_unregister(struct device *dev) -{ - guard(mutex)(&apple_sleep_coprocs_lock); - if (apple_sleep_enclave.dev == dev) - memset(&apple_sleep_enclave, 0, sizeof(apple_sleep_enclave)); -} -EXPORT_SYMBOL_GPL(apple_sleep_enclave_unregister); - -void apple_sleep_coproc_unregister(struct device *dev) -{ - struct apple_sleep_coproc *c; - - guard(mutex)(&apple_sleep_coprocs_lock); - c = apple_sleep_coproc_find(dev); - if (c) - memset(c, 0, sizeof(*c)); -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_unregister); - -void apple_sleep_coproc_set_ready(struct device *dev, bool ready) -{ - struct apple_sleep_coproc *c; - - guard(mutex)(&apple_sleep_coprocs_lock); - c = apple_sleep_coproc_find(dev); - if (!c) - return; - c->ready = ready; - c->stopped = false; -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_ready); - -bool apple_sleep_coproc_stopped(struct device *dev) -{ - struct apple_sleep_coproc *c; - - guard(mutex)(&apple_sleep_coprocs_lock); - c = apple_sleep_coproc_find(dev); - return c && c->stopped; -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_stopped); - -bool apple_sleep_commit_stops_coprocs(void) -{ - return READ_ONCE(apple_sleep) && pm_suspend_via_firmware(); -} -EXPORT_SYMBOL_GPL(apple_sleep_commit_stops_coprocs); - /* - * The last thing that can fail on the way to the commit: no CPU but this one is up and - * interrupts are off, so no lock can be held. A coprocessor stopped here comes back only - * through a real sleep, so every refusal has to be behind this, and nothing after it may - * go back. In the order measured on macOS, on ANS itself: whatever its driver does last, - * status and control read, then 0 written twice. + * The dry run level of the suspend in progress, for a SoC's part that has dry runs: every + * sleep step that can be taken back runs, and the sleep is refused before the commit. 0 + * otherwise. */ -static void apple_sleep_coproc_run_off(struct apple_sleep_coproc *c) -{ - /* Its own last stop cleared its run bits. */ - if (!c->asc) { - c->stopped = true; - return; - } - c->status = readl(c->asc + ASC_CPU_STATUS); - writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); - writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); - c->stopped = true; -} +static unsigned int apple_sleep_dry; -/* A power word that a ready coprocessor's last stop lowers: on until the commit. */ -static bool apple_sleep_coproc_lowers(u32 ps) +void apple_sleep_set_dry(unsigned int level) { - int i; - - for (i = 0; i < SLEEP_COPROCS; i++) - if (apple_sleep_coprocs[i].dev && apple_sleep_coprocs[i].ready && - apple_sleep_coprocs[i].first && apple_sleep_coprocs[i].ps == ps) - return true; - return false; + WRITE_ONCE(apple_sleep_dry, level); } -/* The last refusals, each coprocessor's own: nothing has stopped yet. */ -static int apple_sleep_coprocs_check(struct apple_sleep *s) +bool apple_sleep_power_loss(void) { - int i, ret; - - for (i = 0; i < SLEEP_COPROCS; i++) { - struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; - - if (!c->dev || !c->ready || c->stopped || !c->last_check) - continue; - ret = c->last_check(c->data); - if (ret) { - dev_err(s->dev, "refused: %s: %d\n", dev_name(c->dev), ret); - return ret; - } - } - - return 0; + return pm_suspend_via_firmware() && !READ_ONCE(apple_sleep_dry); } +EXPORT_SYMBOL_GPL(apple_sleep_power_loss); /* * The other cores stop at the commit, and one left awake there keeps the engine waiting @@ -1107,46 +766,6 @@ static int apple_sleep_cpus_hold(struct apple_sleep *s) return ret; } -/* A nonzero result must mean no SLEEP was sent; post-send errors are fatal. */ -static int apple_sleep_enclave_stop(struct apple_sleep *s) -{ - int ret; - - if (!apple_sleep_enclave.stop) - return 0; - ret = apple_sleep_enclave.stop(apple_sleep_enclave.data); - if (ret) - dev_err(s->dev, "refused before SEP SLEEP was sent: %d\n", ret); - - return ret; -} - -/* - * In two passes, after the last refusal: from the first stop only a real sleep restarts the - * GPU, so a failure here goes on to the commit. Those registered first stop whole first, - * before macOS's off words drop under them: the GPU withdraws its PMP request there, and the - * PMP has to see that before its own stop. The rest after the match. - */ -static void apple_sleep_coprocs_stop(struct apple_sleep *s, bool first) -{ - int i; - - if (first) - s->pcie_ans_stopped = false; - for (i = 0; i < SLEEP_COPROCS; i++) { - struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; - - if (!c->dev || !c->ready || c->first != first) - continue; - s->pcie_ans_stopped |= c->pcie_ans; - if (c->stopped) - continue; - c->stop_ret = c->last_stop ? c->last_stop(c->data) : 0; - apple_sleep_coproc_run_off(c); - } - dsb(sy); -} - /* MPIDR's cluster is affinity 2, its core affinity 0, as 0.0.n and 1.1.n on T8140. */ static unsigned int sleep_cpu_cluster(u64 mpidr) { @@ -1246,27 +865,14 @@ static void sleep_ps_lower(struct apple_sleep *s, u32 offset) sleep_ps_lower_finish(s, offset, sleep_ps_lower_start(s, offset)); } -/* - * As macOS raises a domain: the target, then AUTO_ENABLE once it reads on. Measured - * byte for byte on ANS and PCIe, for instance 0x0f002300, 0x0f00200f, 0x1f00200f. - */ static int sleep_ps_raise_start(struct apple_sleep *s, u32 offset) { - void __iomem *reg = s->ps + offset; - u32 now; - - writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET, reg); - - return readl_poll_timeout_atomic(reg, now, (now & PS_ACTUAL) == PS_ACTUAL, 1, - PS_OFF_TIMEOUT_US); + return apple_sleep_ps_raise_start(s->ps + offset); } static void sleep_ps_raise_finish(struct apple_sleep *s, u32 offset) { - void __iomem *reg = s->ps + offset; - - writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET | PS_AUTO_ENABLE, - reg); + apple_sleep_ps_raise_finish(s->ps + offset); } static int sleep_ps_raise(struct apple_sleep *s, u32 offset) @@ -1385,7 +991,6 @@ EXPORT_SYMBOL_GPL(apple_sleep_domain_is_down); * target 4 and 0, released with the domain off, then raised again, its DARTs' state kept. The * caller has cleared the run bit, and starts the firmware after. */ -#define PS_BUSY BIT(11) #define PS_RESET BIT(31) #define PS_ENGINE_RESET_4 0x8f000244 #define PS_ENGINE_RESET_0 0x80000000 @@ -1739,20 +1344,8 @@ static int apple_sleep_refuse(struct apple_sleep *s) bool refused = false, idled = false; int i; - for (i = 0; i < SLEEP_COPROCS; i++) { - struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; - - if (!c->dev) - continue; - if (c->ready) { - idled |= c->pcie_ans; - } else { - /* Nothing may lower a coprocessor's domain under its driver. */ - dev_err(s->dev, "refused: %s is running and its driver has not quiesced it\n", - dev_name(c->dev)); - refused = true; - } - } + if (apple_sleep_coprocs_refuse(s->dev, &idled)) + refused = true; if (num_online_cpus() != 1 || (read_cpuid_mpidr() & MPIDR_HWID_BITMASK) != s->mbox->boot_mpidr) { @@ -2097,14 +1690,6 @@ bool apple_sleep_debug_link_cut(void) } EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut); -bool apple_sleep_debug_link_up(void) -{ - struct apple_sleep *s = apple_sleep; - - return s && !s->usb_down_ran && sleep_debug_link_up(s); -} -EXPORT_SYMBOL_GPL(apple_sleep_debug_link_up); - /* * macOS takes the wake for a lid closing during the sleep as a dark one: every driver wakes, * the display stays off, and the root domain sleeps again as soon as it sees the lid closed. @@ -2120,36 +1705,6 @@ void apple_sleep_lid_closed_wake(void) } EXPORT_SYMBOL_GPL(apple_sleep_lid_closed_wake); -/* Read in the SMC's noirq resume. Forgotten as each sleep begins. */ -static int sleep_lid_closed = -ENODATA; - -void apple_sleep_lid_at_wake(bool closed) -{ - WRITE_ONCE(sleep_lid_closed, closed); -} -EXPORT_SYMBOL_GPL(apple_sleep_lid_at_wake); - -int apple_sleep_lid_closed_at_wake(void) -{ - return READ_ONCE(sleep_lid_closed); -} -EXPORT_SYMBOL_GPL(apple_sleep_lid_closed_at_wake); - -/* Set in the SMC's noirq resume. Forgotten as each sleep begins. */ -static int sleep_radio_woke = -ENODATA; - -void apple_sleep_radio_at_wake(bool woke) -{ - WRITE_ONCE(sleep_radio_woke, woke); -} -EXPORT_SYMBOL_GPL(apple_sleep_radio_at_wake); - -int apple_sleep_radio_woke(void) -{ - return READ_ONCE(sleep_radio_woke); -} -EXPORT_SYMBOL_GPL(apple_sleep_radio_woke); - /* * The SMC woke the AP for the port alone, and this sleep cut the debug link: the cut's own * events. Sleep again. The link stays down until tipd rearms it at a full wake, so the next @@ -2297,21 +1852,6 @@ static void apple_sleep_lower_apcie(struct apple_sleep *s) sleep_ps_lower(s, d->ans); } -/* Read-modify-write in table order, with no write where nothing changes. */ -static void sleep_tunable_apply_changed(void __iomem *regs, const struct apple_tunable *t) -{ - size_t i; - - for (i = 0; i < t->sz; i++) { - void __iomem *reg = regs + t->values[i].offset; - u32 old = readl(reg); - u32 val = (old & ~t->values[i].mask) | (t->values[i].value & t->values[i].mask); - - if (val != old) - writel(val, reg); - } -} - /* * The power loss takes the PHY's PLL and lane settings with it, and ANS's link can't * train without them. Applied as macOS does: the PLL table, then the lanes, then the @@ -2321,8 +1861,8 @@ static void sleep_apcie_phy_ip_apply(struct apple_sleep *s) { void __iomem *ctl = s->apcie_phy_ip + APCIE_PHY_IP_PLL_CTL; - sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_pll); - sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_lanes); + apple_sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_pll); + apple_sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_lanes); writel(readl(ctl) & ~APCIE_PHY_IP_PLL_HOLD, ctl); } @@ -2422,13 +1962,9 @@ static void apple_sleep_lower_io(struct apple_sleep *s) static void apple_sleep_set_wd2(struct apple_sleep *s, unsigned int secs) { - writel(0, s->wdt + WDT_WD2_CTL); - writel(0, s->wdt + WDT_WD2_COUNT); - writel(secs * WDT_TICKS_PER_S, s->wdt + WDT_WD2_ALARM); - writel(1, s->wdt + WDT_WD2_CTL); + apple_sleep_wd2_arm(s->wdt, secs); } -#define PS_PARENT_MISSING BIT(11) #define PS_AUTO_STATE GENMASK(27, 24) #define PS_REWAKE_PASSES 2 @@ -2460,7 +1996,7 @@ static int sleep_ps_find(const u32 *all, int n, u32 offset) */ static int sleep_ps_restore(void __iomem *reg, u32 saved) { - u32 keep = (saved & ~(PS_AUTO_ENABLE | PS_AUTO_STATE | PS_PARENT_MISSING | PS_DEV_DISABLE | + u32 keep = (saved & ~(PS_AUTO_ENABLE | PS_AUTO_STATE | PS_BUSY | PS_DEV_DISABLE | PS_FLAGS | PS_ACTUAL)) | (readl(reg) & PS_AUTO_STATE); u32 now; int ret; @@ -2601,21 +2137,9 @@ static void sleep_ps_rewake_report(struct apple_sleep *s) #define T8132_PS_AON_ON_STATE 4 -/* As macOS writes the PMU: the reply's vector masked, then acknowledged, so none is left raised. */ static void sleep_pmu_write(struct apple_sleep *s, u16 reg, u8 val) { - u32 status, enable; - - enable = readl(s->spmi + SPMI_NOTIFY_ENABLE); - writel(enable & ~SPMI_NOTIFY_REPLY, s->spmi + SPMI_NOTIFY_ENABLE); - writel(((u32)reg << 16) | SPMI_CMD_WRITE | - FIELD_PREP(SPMI_CMD_USID, s->soc->pmu_usid), s->spmi + SPMI_CMD); - writel(val, s->spmi + SPMI_CMD); - if (!readl_poll_timeout_atomic(s->spmi + SPMI_STATUS, status, - !(status & SPMI_STATUS_RX_EMPTY), 1, 1000)) - readl(s->spmi + SPMI_REPLY); - writel(1, s->spmi + SPMI_NOTIFY_ACK); - writel(enable, s->spmi + SPMI_NOTIFY_ENABLE); + apple_sleep_pmu_write(s->spmi, s->soc->pmu_usid, reg, val); } /* @@ -3345,19 +2869,19 @@ static int apple_sleep_enter(suspend_state_t state) if (!ret) ret = sleep_display_off(s); if (!ret) - ret = apple_sleep_coprocs_check(s); + ret = apple_sleep_coprocs_check(s->dev); /* Cover the platform's own preparation, after the PM core's early check. */ if (!ret && pm_wakeup_pending()) ret = -EBUSY; if (!ret) - ret = apple_sleep_enclave_stop(s); + ret = apple_sleep_enclave_stop(s->dev); if (ret) { dev_dbg(s->dev, "refused before any commit coprocessor stop: %d\n", ret); spin_table_unhold_parked(); apple_sleep_stray_restore(s); return ret; } - apple_sleep_coprocs_stop(s, true); + apple_sleep_coprocs_stop(true, &s->pcie_ans_stopped); sleep_wake_arm(s); apple_sleep_match(s); @@ -3365,7 +2889,7 @@ static int apple_sleep_enter(suspend_state_t state) /* Acknowledge a leftover done flag, as the DCS word has none on macOS. */ if (s->soc->ack_cluster_dcs && (readl(s->cluster_dcs) & CLUSTER_UP_DONE)) writel(readl(s->cluster_dcs), s->cluster_dcs); - apple_sleep_coprocs_stop(s, false); + apple_sleep_coprocs_stop(false, &s->pcie_ans_stopped); /* Done by now unless no SMC driver is bound. */ apple_sleep_usb_down(s); s->deep_once = READ_ONCE(sleep_deep_requested); @@ -3393,8 +2917,7 @@ static int apple_sleep_begin(suspend_state_t state) { /* The last sleep's cut is its port controller's to take, by its .complete. */ WRITE_ONCE(sleep_debug_link_cut, false); - WRITE_ONCE(sleep_lid_closed, -ENODATA); - WRITE_ONCE(sleep_radio_woke, -ENODATA); + apple_sleep_wake_info_forget(); /* The PM core also calls .begin for s2idle when no s2idle ops are installed. */ if (state != PM_SUSPEND_MEM) return 0; @@ -3904,9 +3427,8 @@ static int apple_sleep_map_mailbox(struct apple_sleep *s) return 0; } -static struct apple_tunable *apple_sleep_apcie_tunable(struct apple_sleep *s, - struct device_node *np, - const char *name, const char *prop) +struct apple_tunable *apple_sleep_apcie_tunable(struct device *dev, struct device_node *np, + const char *name, const char *prop) { int idx = of_property_match_string(np, "reg-names", name); struct apple_tunable *t; @@ -3914,21 +3436,21 @@ static struct apple_tunable *apple_sleep_apcie_tunable(struct apple_sleep *s, if (idx < 0 || of_address_to_resource(np, idx, &res)) return NULL; - t = devm_apple_tunable_parse(s->dev, np, prop, &res); + t = devm_apple_tunable_parse(dev, np, prop, &res); return IS_ERR(t) ? NULL : t; } -static void __iomem *apple_sleep_apcie_block(struct apple_sleep *s, struct device_node *np, - const char *name, const char *prop, - struct apple_tunable **tunable) +void __iomem *apple_sleep_apcie_block(struct device *dev, struct device_node *np, + const char *name, const char *prop, + struct apple_tunable **tunable) { int idx = of_property_match_string(np, "reg-names", name); if (idx < 0) return NULL; if (prop) { - *tunable = apple_sleep_apcie_tunable(s, np, name, prop); + *tunable = apple_sleep_apcie_tunable(dev, np, name, prop); if (!*tunable) return NULL; } @@ -3947,18 +3469,18 @@ static void apple_sleep_map_apcie(struct apple_sleep *s) if (!np) return; - s->apcie_axi = apple_sleep_apcie_block(s, np, "axi2af", "apple,tunable-axi2af", + s->apcie_axi = apple_sleep_apcie_block(s->dev, np, "axi2af", "apple,tunable-axi2af", &s->apcie_axi_tunable); - s->apcie_cio3pll = apple_sleep_apcie_block(s, np, "cio3pll", "apple,tunable-cio3pll", + s->apcie_cio3pll = apple_sleep_apcie_block(s->dev, np, "cio3pll", "apple,tunable-cio3pll", &s->apcie_cio3pll_tunable); - s->apcie_clkgen = apple_sleep_apcie_block(s, np, "clkgen", "apple,tunable-clkgen", + s->apcie_clkgen = apple_sleep_apcie_block(s->dev, np, "clkgen", "apple,tunable-clkgen", &s->apcie_clkgen_tunable); - s->apcie_phy = apple_sleep_apcie_block(s, np, "phy-block", "apple,tunable-phy", + s->apcie_phy = apple_sleep_apcie_block(s->dev, np, "phy-block", "apple,tunable-phy", &s->apcie_phy_tunable); - s->apcie_rc = apple_sleep_apcie_block(s, np, "rc", NULL, NULL); - s->apcie_phy_ip = apple_sleep_apcie_block(s, np, "phy-ip", "apple,tunable-phy-ip-pll", + s->apcie_rc = apple_sleep_apcie_block(s->dev, np, "rc", NULL, NULL); + s->apcie_phy_ip = apple_sleep_apcie_block(s->dev, np, "phy-ip", "apple,tunable-phy-ip-pll", &s->apcie_phy_ip_pll); - s->apcie_phy_ip_lanes = apple_sleep_apcie_tunable(s, np, "phy-ip", + s->apcie_phy_ip_lanes = apple_sleep_apcie_tunable(s->dev, np, "phy-ip", "apple,tunable-phy-ip-auspma"); if (!s->apcie_axi || !s->apcie_cio3pll || !s->apcie_clkgen || !s->apcie_phy || !s->apcie_rc || !s->apcie_phy_ip || !s->apcie_phy_ip_lanes) { @@ -3970,12 +3492,12 @@ static void apple_sleep_map_apcie(struct apple_sleep *s) s->apcie_axi = NULL; } - s->apcie_port = apple_sleep_apcie_block(s, np, "port0", NULL, NULL); - s->apcie_port_phy = apple_sleep_apcie_block(s, np, "phy0", NULL, NULL); + s->apcie_port = apple_sleep_apcie_block(s->dev, np, "port0", NULL, NULL); + s->apcie_port_phy = apple_sleep_apcie_block(s->dev, np, "phy0", NULL, NULL); engine = of_parse_phandle(np, "apple,piodma", 0); if (engine) { - s->apcie_engine = apple_sleep_apcie_block(s, engine, "engine", NULL, NULL); - s->apcie_fabric = apple_sleep_apcie_block(s, engine, "fabric", NULL, NULL); + s->apcie_engine = apple_sleep_apcie_block(s->dev, engine, "engine", NULL, NULL); + s->apcie_fabric = apple_sleep_apcie_block(s->dev, engine, "fabric", NULL, NULL); of_node_put(engine); } of_node_put(np); @@ -4090,9 +3612,22 @@ static int apple_sleep_alloc_books(struct apple_sleep *s) static int apple_sleep_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + const struct apple_sleep_soc *soc = of_device_get_match_data(dev); struct apple_sleep *s; int ret; + /* A SoC with its own sequences also has its own conditions, and its own answers. */ + if (soc && soc->part) { + ret = apple_sleep_registry_bind(soc->part->registry_flags); + if (ret) + return dev_err_probe(dev, ret, "another sleep controller is bound\n"); + ret = soc->part->probe(pdev); + if (ret) + apple_sleep_registry_unbind(); + + return ret; + } + /* Below EL2 a monitor owns all of this. */ if (!is_kernel_in_hyp_mode()) return -ENODEV; @@ -4146,12 +3681,19 @@ static int apple_sleep_probe(struct platform_device *pdev) platform_set_drvdata(pdev, s); - ret = spin_table_set_power_ops(&sleep_cpu_power_ops); + ret = apple_sleep_registry_bind(APPLE_SLEEP_REG_LID_AT_WAKE | + APPLE_SLEEP_REG_USB_WAKE_PORTS); if (ret) + return dev_err_probe(dev, ret, "another sleep controller is bound\n"); + ret = spin_table_set_power_ops(&sleep_cpu_power_ops); + if (ret) { + apple_sleep_registry_unbind(); return dev_err_probe(dev, ret, "cannot control CPU power\n"); + } /* Last: the exported helpers take it as a probed driver, and a failure frees s. */ apple_sleep = s; + apple_sleep_registry_ready(); /* Deep becomes the default sleep state, unless mem_sleep_default= chose another. */ suspend_set_ops(&apple_sleep_ops); @@ -4579,6 +4121,12 @@ static const struct apple_sleep_soc t8132_sleep_soc = { .nwake_names = ARRAY_SIZE(t8132_wake_names), .wake_last_word = GENMASK(20, 0), }; +#if APPLE_SLEEP_T6040 +/* T6040 (MacBook Pro J616s): its own sequences, in sleep-t6040.c. */ +static const struct apple_sleep_soc t6040_sleep_soc = { + .part = &t6040_sleep_part, +}; +#endif /* * Only SoCs whose sleep was measured and tested. On every other the driver doesn't bind, and @@ -4587,13 +4135,31 @@ static const struct apple_sleep_soc t8132_sleep_soc = { static const struct of_device_id apple_sleep_of_match[] = { { .compatible = "apple,t8140-sleep", .data = &t8140_sleep_soc }, { .compatible = "apple,t8132-sleep", .data = &t8132_sleep_soc }, +#if APPLE_SLEEP_T6040 + { .compatible = "apple,t6040-sleep", .data = &t6040_sleep_soc }, +#endif { } }; +static int apple_sleep_restore(struct device *dev) +{ + const struct apple_sleep_soc *soc = of_device_get_match_data(dev); + + if (soc && soc->part && soc->part->restore) + return soc->part->restore(dev); + + return 0; +} + +static const struct dev_pm_ops apple_sleep_pm_ops = { + .restore = pm_sleep_ptr(apple_sleep_restore), +}; + static struct platform_driver apple_sleep_driver = { .driver = { .name = "apple-sleep", .of_match_table = apple_sleep_of_match, + .pm = pm_sleep_ptr(&apple_sleep_pm_ops), .suppress_bind_attrs = true, }, .probe = apple_sleep_probe, diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index ea0b6fe76b34a..9042ebcba7a8f 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -532,6 +532,15 @@ static void apple_spmi_irq_handler(struct irq_desc *desc) chained_irq_exit(chip, desc); } +/* Every interrupt masked, the notification included, and what is latched dropped. */ +static void apple_spmi_irq_reset(struct apple_spmi *spmi) +{ + for (size_t offset = 0; offset < (spmi->hw->num_irqs + 32) / 8; offset += 4) { + writel(0, spmi->regs + SPMI_IRQ_MASK_BASE + offset); + writel(U32_MAX, spmi->regs + SPMI_IRQ_ACK_BASE + offset); + } +} + static void apple_spmi_teardown_irq(void *data) { struct apple_spmi *spmi = data; @@ -556,10 +565,7 @@ static int apple_spmi_init_irq(struct platform_device *pdev, raw_spin_lock_init(&spmi->irq_mask_lock); - for (size_t offset = 0; offset < (spmi->hw->num_irqs + 32) / 8; offset += 4) { - writel(0, spmi->regs + SPMI_IRQ_MASK_BASE + offset); - writel(U32_MAX, spmi->regs + SPMI_IRQ_ACK_BASE + offset); - } + apple_spmi_irq_reset(spmi); spmi->irqd = devm_irq_domain_instantiate(&pdev->dev, &info); if (IS_ERR(spmi->irqd)) @@ -614,6 +620,23 @@ static int apple_spmi_resume_noirq(struct device *dev) return 0; guard(mutex)(&spmi->fifo_lock); + /* + * Firmware and the platform's sleep code use this queue while Linux is + * away. Every command of ours was answered before the suspend, so any + * reply waiting now is someone else's, and would be read as the reply + * to our next command. + */ + if (pm_resume_via_firmware()) { + unsigned int stale = 0; + + while (!(readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) && + stale < 64) { + readl(spmi->regs + spmi->hw->reply); + stale++; + } + if (stale) + dev_info(dev, "dropped %u stale reply words after the wake\n", stale); + } raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); /* Whether the enables survive a real sleep has not been measured. */ if (pm_resume_via_firmware()) { @@ -630,9 +653,6 @@ static int apple_spmi_resume_noirq(struct device *dev) return 0; } -static DEFINE_NOIRQ_DEV_PM_OPS(apple_spmi_pm_ops, apple_spmi_suspend_noirq, - apple_spmi_resume_noirq); - static int apple_spmi_probe(struct platform_device *pdev) { struct apple_spmi *spmi; @@ -688,6 +708,73 @@ static int apple_spmi_probe(struct platform_device *pdev) return 0; } +/* + * After a resume from disk the controller holds the boot kernel's masks and + * pending interrupts, not this kernel's: drop the pending ones and rewrite + * the masks from the cache the mask/unmask paths keep, notification IRQ + * included. + */ +static void apple_spmi_hib_resync_irqs(struct apple_spmi *spmi) +{ + u32 nirq = spmi->hw->num_irqs; + unsigned long flags; + + raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); + /* + * As at probe, drop what is latched: interrupts the boot kernel left + * pending complete a register selection of a device restored now + * before its own answer arrives. The devices read their state afresh. + */ + apple_spmi_irq_reset(spmi); + for (size_t offset = 0; offset < (nirq + 32) / 8; offset += 4) { + u32 val = 0; + + if (offset < nirq / 8) + val = bitmap_read(spmi->irq_mask_cache, offset * 8, 32); + else if (spmi->notify_irq && offset == (nirq / 32) * 4) + val = BIT(nirq % 32); + writel(val, spmi->regs + SPMI_IRQ_MASK_BASE + offset); + } + raw_spin_unlock_irqrestore(&spmi->irq_mask_lock, flags); +} + +/* + * Left alone, the masks stay as the boot kernel left them, everything masked + * but the interrupts of the drivers it loaded, and the IRQ core does not write + * them again (it disables lazily and unmasks only an IRQ it recorded as + * masked). A device whose driver the boot kernel did not load, such as a + * Type-C port controller on nub_spmi_a0/a1, would raise no event after the + * restore. + * + * In the early restore, for every controller alike: one in a PM domain gets no + * restore_noirq of its own, as genpd calls the driver's resume_noirq there + * (outside a wakeup path), and genpd does not wrap the early restore. It still + * runs before the devices on the bus are restored. + */ +static int apple_spmi_restore_early(struct device *dev) +{ + struct apple_spmi *spmi = dev_get_drvdata(dev); + + if (spmi->irq < 0) + return 0; + + /* Commands can run by now: none may have its reply acknowledged away. */ + scoped_guard(mutex, &spmi->fifo_lock) + apple_spmi_hib_resync_irqs(spmi); + dev_dbg(dev, "resume from disk: interrupt masks rewritten in the early restore\n"); + + return 0; +} + +/* + * Suspend-to-RAM parks the notification queue; a resume from disk rewrites the + * masks in the early restore. + */ +static const struct dev_pm_ops apple_spmi_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_spmi_suspend_noirq, apple_spmi_resume_noirq) + .restore_early = pm_sleep_ptr(apple_spmi_restore_early), +}; + static const struct of_device_id apple_spmi_match_table[] = { { .compatible = "apple,t8142-spmi", .data = &apple_spmi_gen4 }, { .compatible = "apple,t8103-spmi", .data = &apple_spmi_gen1 }, diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 3aa62de12e91b..ec4414f3ca762 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -95,6 +95,8 @@ #include #include #include +#include +#include #include #include #include @@ -148,6 +150,11 @@ struct apple_cio_hw { bool skip_run; /* Hand LSTX to the router after RTKit boot, not before it. */ bool lstx_after_boot; + /* + * A stopped router left LSTX high unless it drives the pin low: one + * its firmware released counts as high as well. + */ + bool lstx_released_high; bool strict_fw_ready; /* Keep the links of the host router out of CL states. */ bool no_host_clx; @@ -168,6 +175,15 @@ struct apple_cio_hw { bool gen4_link_recovery; /* Leave the host router's TMU to its firmware; host TMU constants. */ bool host_tmu; + /* + * The router runs only over its Type-C PHY in CIO mode ("phys") + * and tells the PHY when it starts and stops. + */ + bool cio_phy; + /* Power the router off for every system sleep, hibernation included. */ + bool sleep_stop; + /* Let a Thunderbolt 3 (not USB4) link settle after RTKit boot. */ + bool tbt_settle; const struct apple_dpin_policy *dp; }; @@ -234,17 +250,38 @@ static const struct apple_cio_hw apple_cio_t6050_hw = { .dp = &apple_dpin_m5, }; +/* + * T6040/T6041 (M4 Pro/Max) routers are Type 7 (acio,v2) as on T8142, without + * per-direction NHI fabric tables. The boot loader leaves the CIO firmware in + * the router's SRAM, and the CIO reconfiguration this driver requests through + * its reset restarts the CPU from it: the firmware is not loaded again and + * RUN is never written. Once RTKit is up, a Thunderbolt 3 (not USB4) link + * gets 400 ms to settle, LSTX is handed to the router, and the firmware state + * is read until it is 1 (bit 31 set: the firmware asserted, with a code). The + * router is powered off for every system sleep, with or without a cable, and + * started cold on wake. The host links stay in CL0 until CL states are + * qualified. + */ static const struct apple_cio_hw apple_cio_t6040_hw = { .mxwrap = true, .type7 = true, .lstx_explicit = true, .skip_run = true, .lstx_after_boot = true, + .lstx_released_high = true, .strict_fw_ready = true, .no_host_clx = true, + .cio_phy = true, + .sleep_stop = true, + .tbt_settle = true, .dp = &apple_dpin_disabled, }; +/* Settle time of a Thunderbolt 3 or older link once the router's RTKit is up */ +#define APPLE_CIO_TBT_SETTLE_MS 400 +/* Upper bound for a deferred sleep stop before the suspend is aborted. */ +#define APPLE_CIO_SLEEP_STOP_TIMEOUT_MS 60000 + #define APPLE_CIO_NHI_HOP_COUNT 0x0 #define APPLE_CIO_NHI_HOP_COUNT_MASK GENMASK(9, 0) @@ -337,6 +374,19 @@ MODULE_PARM_DESC(cio_s2r, "Enable J713 attached-dock system sleep (default: on)" struct apple_dpin_binding; +/* + * hw->sleep_stop: where the stop of a port with a PCIe-C host stands, which + * the PM notifier leaves to sleep_stop_work. While that work runs it holds + * the lock, and a PCIe endpoint driver it unbinds may wait for the PM notifier + * chain to end. The notifier therefore waits neither for the work nor for the + * lock unless this says that the work is over. + */ +enum apple_cio_sleep_stop { + APPLE_CIO_SLEEP_STOP_NONE, /* not deferred, or seen to be over */ + APPLE_CIO_SLEEP_STOP_QUEUED, /* for the sleep that is under way */ + APPLE_CIO_SLEEP_STOP_ORPHAN, /* that sleep ended, the stop has not */ +}; + /** * struct apple_cio - Apple Converged I/O host router complex * @dev: ACIO device @@ -355,6 +405,11 @@ struct apple_dpin_binding; * @nhi_boot_completion: Signals completion of the NHI bringup * @nhi_boot_status: Status returned by the NHI bringup * @tbt_switch: USB4/Thunderbolt switch triggered by the Type-C PD driver + * @sleep_hibernating: Hibernation is under way; thaw does not reopen the port + * @sleep_stop_work: Stops a port with a PCIe-C host for system sleep + * @sleep_stop_done: Signals that @sleep_stop_work has finished + * @sleep_stop_state: Where @sleep_stop_work stands for the PM notifier + * @phy: Type-C PHY the router runs over (hw->cio_phy) */ struct apple_cio { struct device *dev; @@ -418,6 +473,14 @@ struct apple_cio { } sleep_phase; /* protected by lock */ bool sleep_restart_owed; /* protected by lock */ bool sleep_aborted_early; /* protected by lock: devices never suspended */ + /* hw->sleep_stop: hibernation is under way, thaw does not reopen */ + bool sleep_hibernating; /* PM notifier and complete only */ + /* hw->sleep_stop: the stop of a port with a PCIe-C host runs here */ + struct work_struct sleep_stop_work; + struct completion sleep_stop_done; + atomic_t sleep_stop_state; /* enum apple_cio_sleep_stop */ + /* hw->cio_phy: the Type-C PHY this router runs over */ + struct phy *phy; u32 current_cable_info; u32 target_cable_info; @@ -2394,6 +2457,18 @@ static const struct tb_nhi_ops apple_nhi_ops = { .pci_tunnel_deactivate = apple_nhi_pci_tunnel_deactivate, }; +/* + * hw->cio_phy: the port's USB controller reaches a dock's USB3 through the + * router's host USB3 adapter, so the PHY points its PIPE at the router while + * such a tunnel is up and back before the tunnel goes. + */ +static int apple_nhi_usb3_tunnel_changed(struct tb_nhi *nhi, bool on) +{ + struct apple_cio *acio = nhi_to_anhi(nhi)->acio; + + return apple_t6040_cio_usb3_tunnel(acio->phy, on); +} + static const struct tb_nhi_ring_layout apple_nhi_ring_layout = { .tx_desc_base = APPLE_CIO_NHI_TXRING_DESC_BASE, .rx_desc_base = APPLE_CIO_NHI_RXRING_DESC_BASE, @@ -2517,6 +2592,8 @@ static int apple_nhi_probe(struct platform_device *pdev) ret = apple_nhi_cio_bind(anhi); if (ret) goto err; + if (acio->hw->cio_phy) + anhi->ops.usb3_tunnel_changed = apple_nhi_usb3_tunnel_changed; anhi->nhi.ops = &anhi->ops; anhi->nhi.ring_layout = &apple_nhi_ring_layout; anhi->nhi.iobase = anhi->nhi_base; @@ -3068,6 +3145,12 @@ static void apple_cio_stop(struct apple_cio *acio) lockdep_assert_held(&acio->lock); if (!acio->current_cable_info) return; + /* + * Domain removal frees the tunnels without deactivating them one by + * one: give the USB controller its PIPE back before the router goes. + */ + if (acio->hw->cio_phy) + apple_t6040_cio_usb3_tunnel(acio->phy, false); apple_cio_remove_children(acio); /* Try to shut down and power off the co-processor gracefully */ @@ -3079,11 +3162,18 @@ static void apple_cio_stop(struct apple_cio *acio) acio->rtk = NULL; apple_cio_cpu_halt(acio); - /* High only if the router left it driven high with the cable still in */ - if (acio->hw->lstx_after_boot) + if (acio->hw->lstx_released_high) { + u32 lstx = readl(acio->rc_base + APPLE_CIO_M3_CTRL); + + /* High unless the router drives it low, with the cable still in */ + apple_cio_lstx_hold(acio, acio->target_cable_info && + lstx != APPLE_CIO_LSTX_CTRL_ENABLE); + } else if (acio->hw->lstx_after_boot) { + /* High only if the router left it driven high with the cable still in */ apple_cio_lstx_hold(acio, acio->target_cable_info && (readl(acio->rc_base + APPLE_CIO_M3_CTRL) & APPLE_CIO_LSTX_CTRL_LEVEL)); + } /* Finally, remove the links to the PD domains to power everything off */ for (i = 0; i < acio->pd_list->num_pds; i++) { @@ -3096,6 +3186,9 @@ static void apple_cio_stop(struct apple_cio *acio) acio->current_cable_info = 0; acio->resume_link_failed = false; acio->link_lost_restart = false; + /* ACIO is off: the PHY may leave CIO now (unplug, removal, sleep). */ + if (acio->hw->cio_phy) + apple_t6040_cio_router_stopped(acio->phy); } /* @@ -3133,18 +3226,11 @@ static bool apple_cio_recovering(struct apple_cio *acio) return acio->resume_recovery_active; } -/** - * apple_cio_start - Power up and start the ACIO complex - * @acio: ACIO complex to start - * - * The Type-C PHY must already be configured for USB4 or Thunderbolt. Power up - * ACIO, boot its RTKit co-processor, populate its child devices and wait for - * the NHI to register the USB4 domain. All completed steps are unwound on - * failure. The caller must hold @acio->lock. - * - * Return: 0 on success or a negative error code on failure. +/* + * Power ACIO up and start it: all of apple_cio_start() but the handshake with + * the Type-C PHY of a hw->cio_phy router. */ -static int apple_cio_start(struct apple_cio *acio) +static int apple_cio_power_up(struct apple_cio *acio) { struct device_link *link; int i, ret, attempts; @@ -3268,6 +3354,11 @@ static int apple_cio_start(struct apple_cio *acio) goto err_free_rtkit; } + /* A Thunderbolt 3 or legacy link gets its settle time here, USB4 none. */ + if (acio->hw->tbt_settle && + (acio->target_cable_info & TB_VSE_CAP_APPLE_CABLE_INFO_TBT2_3)) + msleep(APPLE_CIO_TBT_SETTLE_MS); + /* * The router takes LSTX over at the level it last drove, low after a * cold start, and then gets its pin back from the AOP GPIO controller. @@ -3361,6 +3452,46 @@ static int apple_cio_start(struct apple_cio *acio) return ret; } +/** + * apple_cio_start - Power up and start the ACIO complex + * @acio: ACIO complex to start + * + * The Type-C PHY must already be configured for USB4 or Thunderbolt. Power up + * ACIO, boot its RTKit co-processor, populate its child devices and wait for + * the NHI to register the USB4 domain. All completed steps are unwound on + * failure. The caller must hold @acio->lock. + * + * Return: 0 on success or a negative error code on failure. + */ +static int apple_cio_start(struct apple_cio *acio) +{ + int ret; + + lockdep_assert_held(&acio->lock); + if (!acio->hw->cio_phy) + return apple_cio_power_up(acio); + + /* + * The PHY must be in CIO, tuned by its firmware, before ACIO powers up. + * The Type-C controller sets the mux first and retries on its next event. + */ + ret = apple_t6040_cio_state(acio->phy); + if (ret != 1) { + ret = ret < 0 ? ret : -EAGAIN; + dev_dbg(acio->dev, "Type-C PHY is not in CIO mode (%d), not starting\n", ret); + return ret; + } + ret = apple_t6040_cio_router_started(acio->phy); + if (ret) { + dev_err(acio->dev, "Type-C PHY refused the router: %d\n", ret); + return ret; + } + ret = apple_cio_power_up(acio); + if (ret) + apple_t6040_cio_router_stopped(acio->phy); + return ret; +} + static int apple_cio_check_connection(struct apple_cio *acio) { struct platform_device *pdev; @@ -3549,6 +3680,18 @@ static int apple_cio_prepare(struct device *dev) { struct apple_cio *acio = dev_get_drvdata(dev); + /* + * A sleep stop deferred from the PM notifier must be over before any + * device suspends: the PHY refuses to sleep under a running router. + * Notifiers are done and the PM core holds no lock the stop needs. + */ + if (atomic_read(&acio->sleep_stop_state) == APPLE_CIO_SLEEP_STOP_QUEUED && + !wait_for_completion_timeout(&acio->sleep_stop_done, + msecs_to_jiffies(APPLE_CIO_SLEEP_STOP_TIMEOUT_MS))) { + dev_err(dev, "router did not stop for system sleep, aborting it\n"); + return -EBUSY; + } + /* Drain before taking lock; the worker acquires it to check the link. */ cancel_delayed_work_sync(&acio->resume_link_work); cancel_delayed_work_sync(&acio->link_lost_work); @@ -3566,7 +3709,8 @@ static void apple_cio_complete(struct device *dev) struct apple_cio *acio = dev_get_drvdata(dev); guard(mutex)(&acio->lock); - if (acio->sleep_phase == APPLE_CIO_SLEEP_CLOSED) + /* Thaw before the image is written is not a resume: stay closed. */ + if (acio->sleep_phase == APPLE_CIO_SLEEP_CLOSED && !acio->sleep_hibernating) acio->sleep_phase = APPLE_CIO_SLEEP_RESUMED; if (!READ_ONCE(acio->resume_link_pending)) return; @@ -3756,14 +3900,35 @@ static void apple_cio_sleep_close_owed(struct apple_cio *acio) /* * Runs when no post-resume revalidation has restarted the port: always - * after an aborted suspend, and as a fallback after a real one. Ask CD321x - * for a fresh read instead of starting from the state recorded before sleep. + * after an aborted suspend, and as a fallback after a real one; for a + * sleep_stop router also at once when a cable was only recorded while the + * port was closed. Ask CD321x for a fresh read instead of starting from the + * state recorded before sleep. */ static void apple_cio_sleep_restart_work(struct work_struct *work) { struct apple_cio *acio = container_of(to_delayed_work(work), struct apple_cio, sleep_restart_work); + /* + * A sleep that was abandoned before this device's .prepare may have + * left its deferred stop running: wait for it here, outside the PM + * notifier chain. If another sleep has begun by then, the stop is + * that sleep's, whose PM_POST_* queues this work again. + */ + switch (atomic_read(&acio->sleep_stop_state)) { + case APPLE_CIO_SLEEP_STOP_QUEUED: + return; + case APPLE_CIO_SLEEP_STOP_ORPHAN: + flush_work(&acio->sleep_stop_work); + if (atomic_cmpxchg(&acio->sleep_stop_state, + APPLE_CIO_SLEEP_STOP_ORPHAN, + APPLE_CIO_SLEEP_STOP_NONE) != + APPLE_CIO_SLEEP_STOP_ORPHAN) + return; + break; + } + scoped_guard(mutex, &acio->lock) { acio->sleep_aborted_early = false; if (!acio->sleep_restart_owed) @@ -3780,18 +3945,125 @@ static void apple_cio_sleep_restart_work(struct work_struct *work) typec_thunderbolt_switch_notify_restore(acio->tbt_switch); } +/* + * hw->sleep_stop: the router goes down for every system sleep, + * whether the sleep is s2idle, a firmware-driven suspend or hibernation. + * Nothing of the router survives that is used again: the next start is a cold + * one, CIO reconfiguration (which restarts the firmware the boot loader left in + * the router's SRAM) and RTKit boot, after the Type-C controller has reapplied + * the cable. + * Until the sleep is over, cable changes are only recorded. Only a port that + * had a cable owes a restart: one plugged in during the sleep is found by the + * Type-C controller, which reads every port again after a sleep. + * Called with acio->lock held. + */ +static void apple_cio_sleep_stop_locked(struct apple_cio *acio) +{ + lockdep_assert_held(&acio->lock); + + acio->sleep_phase = APPLE_CIO_SLEEP_CLOSED; + if (acio->removing || !acio->current_cable_info) + return; + + WRITE_ONCE(acio->sleep_restart_owed, true); + /* Close PCIe admission; the next NHI probe reopens it. */ + scoped_guard(mutex, &acio->pcie_tunnel_lock) + acio->pcie_tunnel_stopping = true; + dev_dbg(acio->dev, "powering off router for system sleep (cable info 0x%x)\n", + acio->current_cable_info); + apple_cio_stop(acio); +} + +static void apple_cio_sleep_stop_work(struct work_struct *work) +{ + struct apple_cio *acio = container_of(work, struct apple_cio, sleep_stop_work); + + scoped_guard(mutex, &acio->lock) + apple_cio_sleep_stop_locked(acio); + complete_all(&acio->sleep_stop_done); +} + +static void apple_cio_sleep_stop(struct apple_cio *acio, bool hibernate) +{ + bool defer; + + acio->sleep_hibernating = hibernate; + /* + * The stop of a sleep that was abandoned may still be running. It + * stops the router for this sleep as well, and the restart work + * behind it stands down; neither is waited for here. + */ + if (atomic_cmpxchg(&acio->sleep_stop_state, APPLE_CIO_SLEEP_STOP_ORPHAN, + APPLE_CIO_SLEEP_STOP_QUEUED) == APPLE_CIO_SLEEP_STOP_ORPHAN) + return; + + /* Nothing from an earlier sleep may restart the port during this one. */ + cancel_delayed_work_sync(&acio->sleep_restart_work); + cancel_delayed_work_sync(&acio->resume_link_work); + + scoped_guard(mutex, &acio->lock) { + acio->sleep_aborted_early = false; + /* + * PCIe endpoints may unregister PM notifiers on removal, which + * would deadlock inside this notifier chain. A port with a PCIe-C + * host stops from a worker instead. That may still run while + * tasks freeze; this device's .prepare waits for it before any + * device suspends. + */ + scoped_guard(mutex, &acio->pcie_tunnel_lock) + defer = acio->current_cable_info && acio->pcie_tunnel_populated; + if (!defer) { + apple_cio_sleep_stop_locked(acio); + return; + } + } + + reinit_completion(&acio->sleep_stop_done); + atomic_set(&acio->sleep_stop_state, APPLE_CIO_SLEEP_STOP_QUEUED); + queue_work(system_dfl_wq, &acio->sleep_stop_work); +} + +/* + * PM_POST_*: whether the notifier may take the lock. A deferred stop is over + * once this device's .prepare has run. A sleep abandoned before that can find + * it still running, and waiting for it here would keep a PCIe endpoint that + * unregisters its PM notifier, and so the stop itself, from finishing until + * the wait gives up. Such a stop is left to the restart work. + */ +static bool apple_cio_sleep_stop_over(struct apple_cio *acio) +{ + if (atomic_read(&acio->sleep_stop_state) == APPLE_CIO_SLEEP_STOP_NONE) + return true; + if (completion_done(&acio->sleep_stop_done)) { + atomic_set(&acio->sleep_stop_state, APPLE_CIO_SLEEP_STOP_NONE); + return true; + } + atomic_set(&acio->sleep_stop_state, APPLE_CIO_SLEEP_STOP_ORPHAN); + return false; +} + static int apple_cio_pm_notify(struct notifier_block *nb, unsigned long event, void *unused) { struct apple_cio *acio = container_of(nb, struct apple_cio, pm_notifier); + bool stop = acio->hw->sleep_stop; bool owed, early; /* Teardown off and no restart owed from an earlier sleep: nothing to do. */ - if (!READ_ONCE(s2idle_teardown) && !READ_ONCE(acio->sleep_restart_owed)) + if (!stop && !READ_ONCE(s2idle_teardown) && !READ_ONCE(acio->sleep_restart_owed)) return NOTIFY_DONE; switch (event) { + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + if (!stop) + break; + fallthrough; case PM_SUSPEND_PREPARE: + if (stop) { + apple_cio_sleep_stop(acio, event != PM_SUSPEND_PREPARE); + break; + } /* Nothing from an earlier sleep may restart the port during this one. */ cancel_delayed_work_sync(&acio->sleep_restart_work); apple_cio_sleep_close_owed(acio); @@ -3799,15 +4071,35 @@ static int apple_cio_pm_notify(struct notifier_block *nb, unsigned long event, if (READ_ONCE(s2idle_teardown)) apple_cio_sleep_teardown(acio); break; + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + if (!stop) + break; + fallthrough; case PM_POST_SUSPEND: + acio->sleep_hibernating = false; + if (!apple_cio_sleep_stop_over(acio)) { + queue_delayed_work(system_dfl_wq, &acio->sleep_restart_work, 0); + break; + } scoped_guard(mutex, &acio->lock) { - /* Aborted before device suspend: complete never ran. */ + /* + * Aborted before device suspend: complete never ran. After + * hibernation complete did not reopen either, and the cable + * state the Type-C controller reapplied on restore was only + * recorded: ask for it again now. + */ if (acio->sleep_phase == APPLE_CIO_SLEEP_CLOSED) { acio->sleep_phase = APPLE_CIO_SLEEP_RESUMED; acio->sleep_aborted_early = true; } - owed = acio->sleep_restart_owed; early = acio->sleep_aborted_early; + /* A cable recorded while closed has nobody else to start it. */ + if (stop && acio->target_cable_info && !acio->current_cable_info) { + WRITE_ONCE(acio->sleep_restart_owed, true); + early = true; + } + owed = acio->sleep_restart_owed; } /* Start from a worker: probing children inside this chain can deadlock. */ if (owed) @@ -3853,6 +4145,8 @@ static int apple_cio_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&acio->resume_link_work, apple_cio_resume_link_work); INIT_DELAYED_WORK(&acio->link_lost_work, apple_cio_link_lost_work); INIT_DELAYED_WORK(&acio->sleep_restart_work, apple_cio_sleep_restart_work); + INIT_WORK(&acio->sleep_stop_work, apple_cio_sleep_stop_work); + init_completion(&acio->sleep_stop_done); init_completion(&acio->nhi_boot_completion); acio->dev = &pdev->dev; acio->np = dev->of_node; @@ -3860,6 +4154,13 @@ static int apple_cio_probe(struct platform_device *pdev) if (!acio->hw) return -EINVAL; + if (acio->hw->cio_phy) { + acio->phy = devm_phy_get(dev, "cio"); + if (IS_ERR(acio->phy)) + return dev_err_probe(dev, PTR_ERR(acio->phy), + "Missing or unbound Type-C PHY (phys, \"cio\")\n"); + } + acio->type7 = acio->hw->type7; if (acio->type7 && of_property_count_u32_elems(acio->np, "apple,tunable-rc") <= 0) @@ -4100,6 +4401,7 @@ static void apple_cio_remove(struct platform_device *pdev) apple_pcie_tunnel_unregister_notifier(&acio->pcie_notifier); unregister_pm_notifier(&acio->pm_notifier); disable_delayed_work_sync(&acio->sleep_restart_work); + disable_work_sync(&acio->sleep_stop_work); disable_delayed_work_sync(&acio->resume_link_work); disable_delayed_work_sync(&acio->link_lost_work); diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index 6a7461ed5602e..4198cf596ef26 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -110,6 +110,9 @@ struct tb_nhi_ring_layout { * connected to a lane adapter of the host router is unplugged. * Lets the NHI recover a port whose link went down while the * cable stays attached. + * @usb3_tunnel_changed: A USB3 tunnel from a host router USB3 adapter is + * now active (@on, after activation) or about to be + * deactivated (!@on). Called with the domain lock held. * @init_interrupts: NHI specific interrupt initialization hook * @router_sleep: Enter host-router sleep with the control channel running * @router_wake: Start the host router's links again after its controller restarted @@ -141,6 +144,7 @@ struct tb_nhi_ops { bool (*add_links)(struct tb_nhi *nhi); bool (*is_present)(struct tb_nhi *nhi); void (*link_lost)(struct tb_nhi *nhi); + int (*usb3_tunnel_changed)(struct tb_nhi *nhi, bool on); void (*dp_tunnel_changed)(struct tb_nhi *nhi, u8 in_port, u64 generation, bool active); bool (*dp_tunnel_awaits_display)(struct tb_nhi *nhi, struct tb_port *in); int (*init_interrupts)(struct tb_nhi *nhi); diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 25986e65f3fda..0e9df279fdc2f 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -948,6 +948,21 @@ static void tb_reclaim_usb3_bandwidth(struct tb *tb, struct tb_port *src_port, tb_tunnel_reclaim_available_bandwidth(tunnel, &available_up, &available_down); } +/* Tell the host interface when a USB3 tunnel from the host router changes. */ +static void tb_usb3_host_tunnel_changed(struct tb_tunnel *tunnel, bool on) +{ + const struct tb_nhi_ops *ops = tunnel->tb->nhi->ops; + int ret; + + if (!tb_tunnel_is_usb3(tunnel) || tb_route(tunnel->src_port->sw) || + !ops || !ops->usb3_tunnel_changed) + return; + ret = ops->usb3_tunnel_changed(tunnel->tb->nhi, on); + if (ret) + tb_tunnel_warn(tunnel, "host USB3 %s failed: %d\n", + on ? "attach" : "detach", ret); +} + static int tb_tunnel_usb3(struct tb *tb, struct tb_switch *sw) { struct tb_switch *parent = tb_switch_parent(sw); @@ -1026,6 +1041,7 @@ static int tb_tunnel_usb3(struct tb *tb, struct tb_switch *sw) } list_add_tail(&tunnel->list, &tcm->tunnel_list); + tb_usb3_host_tunnel_changed(tunnel, true); if (tb_route(parent)) tb_reclaim_usb3_bandwidth(tb, down, up); @@ -1852,6 +1868,7 @@ static void tb_deactivate_and_free_tunnel(struct tb_tunnel *tunnel) return; tb_dp_tunnel_notify(tunnel, false); + tb_usb3_host_tunnel_changed(tunnel, false); tb_tunnel_deactivate(tunnel); list_del(&tunnel->list); diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig index 0d5c41f1a7d5d..b1012cac1b153 100644 --- a/drivers/usb/dwc3/Kconfig +++ b/drivers/usb/dwc3/Kconfig @@ -242,6 +242,9 @@ config USB_DWC3_APPLE_KUNIT_TEST Test the helper with which the Apple Silicon DWC3 glue chooses the USB2 PHY mode it selects before the core leaves reset: device mode, never host mode, and nothing for other states. + Also test its decisions for a deep sleep with the host kept in + place: which ports may, and what the host's resume does for + each thing the PHY can say of the sleep. The tests need no Apple hardware or platform support. They are built under USB_DWC3, so UML without DMA emulation cannot run them. diff --git a/drivers/usb/dwc3/dwc3-apple-state-test.c b/drivers/usb/dwc3/dwc3-apple-state-test.c index 69abb0f32653f..cf86153b55e64 100644 --- a/drivers/usb/dwc3/dwc3-apple-state-test.c +++ b/drivers/usb/dwc3/dwc3-apple-state-test.c @@ -38,12 +38,92 @@ static void dwc3_apple_non_role_test(struct kunit *test) } } -/* A role that arrives during a sleep decides what comes back after it. */ +/* A role that arrives during a sleep decides what comes back; a host-only port has no device. */ static void dwc3_apple_role_during_sleep_test(struct kunit *test) { - KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_HOST), DWC3_APPLE_HOST); - KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_DEVICE), DWC3_APPLE_DEVICE); - KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_NONE), DWC3_APPLE_NO_CABLE); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_HOST, false), DWC3_APPLE_HOST); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_DEVICE, false), + DWC3_APPLE_DEVICE); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_NONE, false), + DWC3_APPLE_NO_CABLE); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_HOST, true), DWC3_APPLE_HOST); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_NONE, true), + DWC3_APPLE_NO_CABLE); + KUNIT_EXPECT_EQ(test, dwc3_apple_state_for_role(USB_ROLE_DEVICE, true), + DWC3_APPLE_NO_CABLE); +} + +/* A fixed hub's host may always be kept; a Type-C port's only by a PHY that restarts it. */ +static void dwc3_apple_sleep_may_keep_test(struct kunit *test) +{ + /* fixed hub (T8140) */ + KUNIT_EXPECT_TRUE(test, dwc3_apple_sleep_may_keep(true, false, DWC3_APPLE_HOST, + false, false)); + /* Type-C port with the T6040 PHY, in host mode */ + KUNIT_EXPECT_TRUE(test, dwc3_apple_sleep_may_keep(false, true, DWC3_APPLE_HOST, + false, false)); + /* any other Type-C port stops */ + KUNIT_EXPECT_FALSE(test, dwc3_apple_sleep_may_keep(false, false, DWC3_APPLE_HOST, + false, false)); + /* no host, no core, a glue that goes away, a core that didn't come back */ + KUNIT_EXPECT_FALSE(test, dwc3_apple_sleep_may_keep(false, true, DWC3_APPLE_NO_CABLE, + false, false)); + KUNIT_EXPECT_FALSE(test, dwc3_apple_sleep_may_keep(false, true, + DWC3_APPLE_PROBE_PENDING, false, false)); + KUNIT_EXPECT_FALSE(test, dwc3_apple_sleep_may_keep(false, true, DWC3_APPLE_DEVICE, + false, false)); + KUNIT_EXPECT_FALSE(test, dwc3_apple_sleep_may_keep(false, true, DWC3_APPLE_HOST, + true, false)); + KUNIT_EXPECT_FALSE(test, dwc3_apple_sleep_may_keep(true, false, DWC3_APPLE_HOST, + false, true)); +} + +/* The core's mark is read only where the core may be touched and wasn't reset. */ +static void dwc3_apple_sleep_reads_mark_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, dwc3_apple_sleep_reads_mark(0)); + KUNIT_EXPECT_TRUE(test, dwc3_apple_sleep_reads_mark(-EBUSY)); + KUNIT_EXPECT_TRUE(test, dwc3_apple_sleep_reads_mark(APPLE_ATCPHY_SLEEP_POWER_LOST)); + KUNIT_EXPECT_FALSE(test, dwc3_apple_sleep_reads_mark(APPLE_ATCPHY_SLEEP_RESET)); + KUNIT_EXPECT_FALSE(test, dwc3_apple_sleep_reads_mark(-ENODEV)); + KUNIT_EXPECT_FALSE(test, dwc3_apple_sleep_reads_mark(-EIO)); + KUNIT_EXPECT_FALSE(test, dwc3_apple_sleep_reads_mark(-EOPNOTSUPP)); +} + +/* A PHY that hibernated the core (T8140): the mark and the always-on side decide. */ +static void dwc3_apple_wake_hibernated_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, dwc3_apple_sleep_wake_plan(0, true), DWC3_APPLE_WAKE_AS_IS); + KUNIT_EXPECT_EQ(test, dwc3_apple_sleep_wake_plan(0, false), DWC3_APPLE_WAKE_RESTORE); + KUNIT_EXPECT_EQ(test, dwc3_apple_sleep_wake_plan(-EBUSY, true), DWC3_APPLE_WAKE_AS_IS); + KUNIT_EXPECT_EQ(test, dwc3_apple_sleep_wake_plan(-EBUSY, false), DWC3_APPLE_WAKE_RESTORE); + KUNIT_EXPECT_EQ(test, dwc3_apple_sleep_wake_plan(APPLE_ATCPHY_SLEEP_POWER_LOST, true), + DWC3_APPLE_WAKE_AS_IS); + KUNIT_EXPECT_EQ(test, dwc3_apple_sleep_wake_plan(APPLE_ATCPHY_SLEEP_POWER_LOST, false), + DWC3_APPLE_WAKE_START_OVER); +} + +/* A PHY that held the core in reset (T6040): the host starts over whatever the mark reads. */ +static void dwc3_apple_wake_reset_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, dwc3_apple_sleep_wake_plan(APPLE_ATCPHY_SLEEP_RESET, false), + DWC3_APPLE_WAKE_PHY_START_OVER); + KUNIT_EXPECT_EQ(test, dwc3_apple_sleep_wake_plan(APPLE_ATCPHY_SLEEP_RESET, true), + DWC3_APPLE_WAKE_PHY_START_OVER); +} + +/* A core that may be unpowered, or whose PHY didn't come back, is not resumed. */ +static void dwc3_apple_wake_failed_test(struct kunit *test) +{ + static const int errors[] = { -ENODEV, -EIO, -ETIMEDOUT, -EOPNOTSUPP, -ENODATA, 3 }; + int i; + + for (i = 0; i < ARRAY_SIZE(errors); i++) { + KUNIT_EXPECT_EQ(test, dwc3_apple_sleep_wake_plan(errors[i], false), + DWC3_APPLE_WAKE_FAILED); + KUNIT_EXPECT_EQ(test, dwc3_apple_sleep_wake_plan(errors[i], true), + DWC3_APPLE_WAKE_FAILED); + } } static struct kunit_case dwc3_apple_state_test_cases[] = { @@ -51,6 +131,11 @@ static struct kunit_case dwc3_apple_state_test_cases[] = { KUNIT_CASE(dwc3_apple_device_selected_before_reset_test), KUNIT_CASE(dwc3_apple_non_role_test), KUNIT_CASE(dwc3_apple_role_during_sleep_test), + KUNIT_CASE(dwc3_apple_sleep_may_keep_test), + KUNIT_CASE(dwc3_apple_sleep_reads_mark_test), + KUNIT_CASE(dwc3_apple_wake_hibernated_test), + KUNIT_CASE(dwc3_apple_wake_reset_test), + KUNIT_CASE(dwc3_apple_wake_failed_test), {} }; @@ -61,5 +146,5 @@ static struct kunit_suite dwc3_apple_state_test_suite = { kunit_test_suite(dwc3_apple_state_test_suite); -MODULE_DESCRIPTION("KUnit tests for the Apple Silicon DWC3 glue USB2 mode choice"); +MODULE_DESCRIPTION("KUnit tests for the Apple Silicon DWC3 glue's USB2 mode and sleep decisions"); MODULE_LICENSE("GPL"); diff --git a/drivers/usb/dwc3/dwc3-apple-state.h b/drivers/usb/dwc3/dwc3-apple-state.h index bb8d827fdec4c..c0f0840916fe5 100644 --- a/drivers/usb/dwc3/dwc3-apple-state.h +++ b/drivers/usb/dwc3/dwc3-apple-state.h @@ -1,13 +1,15 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Apple Silicon DWC3 glue: controller states and the USB2 PHY role choice, - * kept in a header so that the choice can be tested without the hardware. + * Apple Silicon DWC3 glue: controller states, the USB2 PHY role choice and + * the decisions of a sleep with the host kept in place, in a header so that + * they can be tested without the hardware. */ #ifndef __DWC3_APPLE_STATE_H #define __DWC3_APPLE_STATE_H #include +#include #include #include @@ -49,21 +51,100 @@ static inline int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state target } /** - * dwc3_apple_state_for_role() - state a role asks for while the core is stopped for sleep + * dwc3_apple_state_for_role() - state a role asks for while a sleep holds the core * @role: the role asked for + * @host_only: the controller has no device mode * - * Return: the state to bring the controller up in after the sleep. + * Nothing is stopped or started while the sleep lasts, so what dwc3_usb_role_switch_set() + * decides for a running controller is decided here: no device mode on a host-only port. + * + * Return: the state to bring the controller up in once the sleep is over. */ -static inline enum dwc3_apple_state dwc3_apple_state_for_role(enum usb_role role) +static inline enum dwc3_apple_state dwc3_apple_state_for_role(enum usb_role role, + bool host_only) { switch (role) { case USB_ROLE_HOST: return DWC3_APPLE_HOST; case USB_ROLE_DEVICE: - return DWC3_APPLE_DEVICE; + return host_only ? DWC3_APPLE_NO_CABLE : DWC3_APPLE_DEVICE; default: return DWC3_APPLE_NO_CABLE; } } +/** + * dwc3_apple_sleep_may_keep() - whether a deep sleep may find the host kept in place + * @fixed_hub: the controller serves a fixed hub + * @phy_restart: a Type-C port whose PHY keeps the host by the core's reset + * @state: the controller's state + * @shutting_down: the glue is going away + * @dead: the core didn't come back from an earlier sleep + * + * The glue's side of the question. The PHY then has the last word, as only it knows what + * runs on the port's lanes (apple_typec_phy_sleep_prepare()). + * + * Return: true if the PHY is to be asked. + */ +static inline bool dwc3_apple_sleep_may_keep(bool fixed_hub, bool phy_restart, + enum dwc3_apple_state state, bool shutting_down, + bool dead) +{ + return (fixed_hub || phy_restart) && state == DWC3_APPLE_HOST && + !shutting_down && !dead; +} + +/** + * dwc3_apple_sleep_reads_mark() - whether the core's mark says anything after the sleep + * @phy: what the PHY says of the sleep (apple_typec_phy_sleep_result()) + * + * The mark is a value written to a scratch register of the core before the sleep. After an + * error the core may be unpowered and must not be read, and a core the PHY took through its + * reset has lost the mark whatever the SoC did. + * + * Return: true if the register may be read and its value counts. + */ +static inline bool dwc3_apple_sleep_reads_mark(int phy) +{ + return !phy || phy == -EBUSY || phy == APPLE_ATCPHY_SLEEP_POWER_LOST; +} + +/* What the host's resume does after a sleep in place, see dwc3_apple_sleep_wake_plan(). */ +enum dwc3_apple_wake { + DWC3_APPLE_WAKE_AS_IS, /* Nothing was lost: xHCI resumes as usual */ + DWC3_APPLE_WAKE_RESTORE, /* The core is set up again, xHCI restores its state */ + DWC3_APPLE_WAKE_START_OVER, /* The core is set up again, xHCI starts over */ + DWC3_APPLE_WAKE_PHY_START_OVER, /* The same, and the PHY comes back after the core */ + DWC3_APPLE_WAKE_FAILED, /* xHCI must not resume, nor anything touch the core */ +}; + +/** + * dwc3_apple_sleep_wake_plan() - what the host's resume does after a sleep in place + * @phy: what the PHY says of the sleep (apple_typec_phy_sleep_result()) + * @core_kept: the core still holds its mark; false where dwc3_apple_sleep_reads_mark() + * says not to read it + * + * A sleep refused before the PHY did anything (-EBUSY) left all as it was. A PHY that took + * the core through hibernation brings it back with its registers (0), or without them if a + * power domain went off under it: xHCI's saved state then still counts unless the core's + * always-on side lost power as well. A PHY that held the core in reset says so, and the + * host starts over whatever the mark reads. + * + * Return: the plan. + */ +static inline enum dwc3_apple_wake dwc3_apple_sleep_wake_plan(int phy, bool core_kept) +{ + switch (phy) { + case APPLE_ATCPHY_SLEEP_RESET: + return DWC3_APPLE_WAKE_PHY_START_OVER; + case APPLE_ATCPHY_SLEEP_POWER_LOST: + return core_kept ? DWC3_APPLE_WAKE_AS_IS : DWC3_APPLE_WAKE_START_OVER; + case 0: + case -EBUSY: + return core_kept ? DWC3_APPLE_WAKE_AS_IS : DWC3_APPLE_WAKE_RESTORE; + default: + return DWC3_APPLE_WAKE_FAILED; + } +} + #endif /* __DWC3_APPLE_STATE_H */ diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index c3b02c3602d0a..c89293aaf482d 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -107,6 +107,36 @@ * everything before the freezer runs, the way a cable disconnect does, and brings it back * once the system has resumed. See dwc3_apple_pm_notify(). * + * A T6040 Type-C port in host mode sleeps in place as well, in another way, as its PHY has no + * hibernation to take the core through and can't bring itself back with interrupts off (it + * boots a coprocessor). Below the xHCI device, which stays with its USB devices, the port goes + * through the stop and the start of a cable session: + * + * - Before the freezer (dwc3_apple_pm_notify()): the PHY says whether the port only does USB. + * The glue's device stops being a syscore device for this sleep, so that its power domain + * goes off before the commit like a stopped controller's and the PM core runs its callbacks + * and orders it after the DARTs and the PHY. xHCI's own wakeup is off for it, so that xHCI + * halts whatever state the sleep ends up in. A role the port controller asks for from here + * on is kept for after the sleep. + * - Device suspend, after xHCI's (dwc3_apple_suspend(), the parent's callback): the PHY takes + * USB3 and USB2 down and the core into reset. This is what leaves the PHY parked at the + * commit. Nothing touches the core from here on. + * - The noirq phases: the power domain goes off and comes back on, and the DARTs with it, + * through genpd, whether or not the SoC lost power in between. + * - Device resume, in xHCI's before it touches the core (dwc3_apple_pre_host_resume()): the + * PHY releases the reset, the core is set up again as at a start, the PHY brings USB2 and + * USB3 back and gets the PIPE, and xHCI is told that its state is lost. It starts over, + * and the USB core reset-resumes the devices (USB persist): they keep their device numbers, + * and a mounted disk its block device. + * - After the thaw (dwc3_apple_pm_notify()): back to a syscore device, and the role the port + * controller asked for meanwhile. + * + * The same steps run when the sleep is refused late or is a dry run. If any of them fails, + * the host is restarted the old way once the system has resumed, and the devices enumerate + * again; a core that may be unpowered is left alone until its reset releases. A port with a + * DisplayPort link, or whose four lanes are DisplayPort's, or in USB4/Thunderbolt stops for + * the sleep as before. + * * Runtime PM (T6040 and T8132, see dwc3_apple_core_probe()): the glue holds a reference from * the reset deassert to the end of the shutdown, so with no cable the controller can * runtime-suspend and its power domain drop, its DARTs (which save and restore their state) and, @@ -145,6 +175,9 @@ * @pre_core_rpm: Runtime PM is the glue's own until the first core bring-up * @dt_dis_u3_susphy: The device tree keeps USB3 PHY suspend off itself * @sleep_ddr: The host votes for DRAM in a sleep while it runs (apple,sleep-ddr) + * @sleep_restart: A Type-C port whose PHY keeps the host through a deep sleep by holding the + * core in reset, see dwc3_apple_suspend() + * @sleep_xhci_wakeup: xHCI's wakeup setting, which is off for such a sleep * @drd_regs: The controller's optional blocks outside dwc3-core (drd-cfg, drd-bulk, drd-ausbc), * mapped for their tunables * @tunables: Controller tunables from the device tree, see dwc3_apple_tunable_list @@ -189,10 +222,15 @@ struct dwc3_apple { struct delayed_work hub_check; /* The core didn't come back from a sleep or a power cycle: nothing may touch it. */ bool sleep_dead; - /* Stopped for a deep sleep: the state to come back to, and the wakeup setting. */ + /* + * Stopped for a deep sleep: the state to come back to, and the wakeup setting. The + * state also for a Type-C port's host kept through it, see dwc3_apple_suspend(). + */ bool sleep_stopped; bool sleep_wakeup; enum dwc3_apple_state sleep_state; + bool sleep_restart; + bool sleep_xhci_wakeup; void __iomem *drd_regs[DWC3_APPLE_NR_DRD_REGS]; struct apple_tunable *tunables[DWC3_APPLE_NR_TUNABLES]; @@ -424,13 +462,33 @@ static void dwc3_apple_set_ptrcap(struct dwc3_apple *appledwc, u32 mode) #define DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY BIT(30) #define DWC3_APPLE_PIPE_PHY_READY_US 100000 +/* + * The T6040 core came up against the dummy PHY: reset its USB3 PIPE side once the PIPE has + * moved (USB2 is not affected). + */ +static void dwc3_apple_usb3_pipe_reset(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + u32 reg; + + reg = dwc3_readl(dwc, DWC3_GUSB3PIPECTL(0)); + dwc3_writel(dwc, DWC3_GUSB3PIPECTL(0), reg | DWC3_GUSB3PIPECTL_PHYSOFTRST); + usleep_range(1000, 1500); + dwc3_writel(dwc, DWC3_GUSB3PIPECTL(0), reg & ~DWC3_GUSB3PIPECTL_PHYSOFTRST); + msleep(100); +} + /* * The core is powered but its registers are at their reset values. Set it up again the way * dwc3_apple_core_start() and dwc3_apple_set_role() do, in place: the PHYs and the xHCI * device stay, so xHCI can restore its saved state, or start over. Runs in the resume, * while nothing else touches the core. + * + * With @phy_down the PHY held the core in reset with its own sides down (T6040). The phys' + * power counts never dropped, so dwc3_core_init() powers nothing on: the PHY brings USB2 and + * USB3 back itself, after the core's init as at a start, where they come up inside it. */ -static int dwc3_apple_core_reinit(struct dwc3_apple *appledwc) +static int dwc3_apple_core_reinit(struct dwc3_apple *appledwc, bool phy_down) { struct dwc3 *dwc = &appledwc->dwc; int i, ret; @@ -447,6 +505,11 @@ static int dwc3_apple_core_reinit(struct dwc3_apple *appledwc) phy_power_off(dwc->usb2_generic_phy[i]); phy_exit(dwc->usb2_generic_phy[i]); } + if (phy_down) { + ret = apple_typec_phy_sleep_power_up(appledwc->usb2_phy); + if (ret) + return ret; + } dwc3_apple_setup_cio(appledwc); dwc3_apple_apply_tunables(appledwc); @@ -459,6 +522,8 @@ static int dwc3_apple_core_reinit(struct dwc3_apple *appledwc) if (ret) dev_warn(appledwc->dev, "USB3 PHY setup failed after the sleep, USB2 only, err=%d\n", ret); + else if (phy_down && appledwc->usb3_phy) + dwc3_apple_usb3_pipe_reset(appledwc); return 0; } @@ -470,43 +535,64 @@ static int dwc3_apple_core_reinit(struct dwc3_apple *appledwc) * restores its saved state, as macOS's does, unless the always-on side lost power too: then * it's told it lost its state (a positive return). A sleep refused before the hibernation * left the core as it was. + * + * A PHY that held the core in reset for the sleep (T6040) has released it by now and comes + * back with the core, in dwc3_apple_core_reinit(): xHCI always starts over then. + * dwc3_apple_sleep_wake_plan() has the cases. */ static int dwc3_apple_pre_host_resume(struct dwc3 *dwc) { struct dwc3_apple *appledwc = to_dwc3_apple(dwc); - bool lost; + bool kept = false; + int phy, ret; u32 val; - int ret; if (!READ_ONCE(appledwc->sleep_in_place)) return 0; - ret = apple_atcphy_sleep_result(appledwc->usb2_phy); - lost = ret == APPLE_ATCPHY_SLEEP_POWER_LOST; - if (ret == -EBUSY || lost) - ret = 0; - if (!ret && appledwc->drd_regs[0] && /* drd-cfg */ - readl_poll_timeout(appledwc->drd_regs[0] + DWC3_APPLE_CFG_STATUS, val, - val & DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY, 100, - DWC3_APPLE_PIPE_PHY_READY_US)) - dev_warn(appledwc->dev, "PIPE PHY not ready after the sleep: %#x\n", val); - if (!ret && dwc3_readl(dwc, DWC3_GUID) != DWC3_APPLE_SLEEP_GUID) { - ret = dwc3_apple_core_reinit(appledwc); - if (!ret && !lost) { - appledwc->sleep_reinit = true; - dev_dbg(appledwc->dev, - "core lost its registers in the sleep: set up again, xHCI restores its state\n"); - return 0; - } - if (!ret) { - appledwc->sleep_reinit = true; - dev_dbg(appledwc->dev, - "core and its always-on side lost power in the sleep: set up again, xHCI starts over\n"); - return 1; - } + phy = apple_typec_phy_sleep_result(appledwc->usb2_phy); + if (dwc3_apple_sleep_reads_mark(phy)) { + if (appledwc->drd_regs[0] && /* drd-cfg */ + readl_poll_timeout(appledwc->drd_regs[0] + DWC3_APPLE_CFG_STATUS, val, + val & DWC3_APPLE_CFG_STATUS_PIPE_PHY_READY, 100, + DWC3_APPLE_PIPE_PHY_READY_US)) + dev_warn(appledwc->dev, "PIPE PHY not ready after the sleep: %#x\n", val); + kept = dwc3_readl(dwc, DWC3_GUID) == DWC3_APPLE_SLEEP_GUID; } - if (ret) - dev_err(appledwc->dev, "not resuming xHCI, the core didn't come back: %d\n", ret); + + switch (dwc3_apple_sleep_wake_plan(phy, kept)) { + case DWC3_APPLE_WAKE_AS_IS: + return 0; + case DWC3_APPLE_WAKE_RESTORE: + ret = dwc3_apple_core_reinit(appledwc, false); + if (ret) + break; + appledwc->sleep_reinit = true; + dev_dbg(appledwc->dev, + "core lost its registers in the sleep: set up again, xHCI restores its state\n"); + return 0; + case DWC3_APPLE_WAKE_START_OVER: + ret = dwc3_apple_core_reinit(appledwc, false); + if (ret) + break; + appledwc->sleep_reinit = true; + dev_dbg(appledwc->dev, + "core and its always-on side lost power in the sleep: set up again, xHCI starts over\n"); + return 1; + case DWC3_APPLE_WAKE_PHY_START_OVER: + ret = dwc3_apple_core_reinit(appledwc, true); + if (ret) + break; + appledwc->sleep_reinit = true; + dev_dbg(appledwc->dev, + "host kept through the sleep: core and PHY set up again, xHCI starts over\n"); + return 1; + case DWC3_APPLE_WAKE_FAILED: + default: + ret = phy < 0 ? phy : -EINVAL; + break; + } + dev_err(appledwc->dev, "not resuming xHCI, the core didn't come back: %d\n", ret); return ret; } @@ -785,22 +871,6 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st return 0; } -/* - * The T6040 core came up against the dummy PHY: reset its USB3 PIPE side once the PIPE has - * moved, as the tested bring-up does (USB2 is not affected). - */ -static void dwc3_apple_usb3_pipe_reset(struct dwc3_apple *appledwc) -{ - struct dwc3 *dwc = &appledwc->dwc; - u32 reg; - - reg = dwc3_readl(dwc, DWC3_GUSB3PIPECTL(0)); - dwc3_writel(dwc, DWC3_GUSB3PIPECTL(0), reg | DWC3_GUSB3PIPECTL_PHYSOFTRST); - usleep_range(1000, 1500); - dwc3_writel(dwc, DWC3_GUSB3PIPECTL(0), reg & ~DWC3_GUSB3PIPECTL_PHYSOFTRST); - msleep(100); -} - /* Select the role in the core and route the USB3 PHY, before xhci or the gadget is started */ static int dwc3_apple_set_role(struct dwc3_apple *appledwc, enum dwc3_apple_state state) { @@ -1128,9 +1198,14 @@ static int dwc3_usb_role_switch_set(struct usb_role_switch *sw, enum usb_role ro if (appledwc->fixed_hub) return 0; - /* Stopped for a sleep: this is what comes back after it. */ - if (appledwc->sleep_stopped) { - appledwc->sleep_state = dwc3_apple_state_for_role(role); + /* + * Held by a sleep: this is what comes back after it. Stopped for it, or kept through it + * with the core in reset for some of it, nothing stops or starts until the sleep is + * over, see dwc3_apple_sleep_restart_end(). Nor while the core may be unpowered after + * one: dwc3_apple_recover() starts what is asked for here. + */ + if (appledwc->sleep_stopped || appledwc->sleep_in_place || appledwc->sleep_dead) { + appledwc->sleep_state = dwc3_apple_state_for_role(role, appledwc->host_only); return 0; } @@ -1188,7 +1263,9 @@ static enum usb_role dwc3_usb_role_switch_get(struct usb_role_switch *sw) guard(mutex)(&appledwc->lock); - switch (appledwc->sleep_stopped ? appledwc->sleep_state : appledwc->state) { + switch (appledwc->sleep_stopped || (appledwc->sleep_restart && + (appledwc->sleep_in_place || appledwc->sleep_dead)) ? + appledwc->sleep_state : appledwc->state) { case DWC3_APPLE_HOST: return USB_ROLE_HOST; case DWC3_APPLE_DEVICE: @@ -1218,6 +1295,20 @@ static int dwc3_apple_setup_role_switch(struct dwc3_apple *appledwc) return 0; } +/* + * What a host lost in a sleep in place comes back as, once it is stopped: a host, or on a + * Type-C port whatever the port controller has asked for since the sleep began. + */ +static void dwc3_apple_restart_lost(struct dwc3_apple *appledwc) +{ + enum dwc3_apple_state state = appledwc->sleep_restart ? appledwc->sleep_state : + DWC3_APPLE_HOST; + + lockdep_assert_held(&appledwc->lock); + if (state == DWC3_APPLE_HOST || state == DWC3_APPLE_DEVICE) + dwc3_apple_start_after_sleep(appledwc, state); +} + /* * A core that may be unpowered gets one more power cycle, as macOS's Off and cold start, and * the host starts over once it's back. Until then nothing may touch it, and every sleep is @@ -1228,7 +1319,7 @@ static bool dwc3_apple_recover(struct dwc3_apple *appledwc) int ret; lockdep_assert_held(&appledwc->lock); - ret = apple_atcphy_sleep_recover(appledwc->usb2_phy); + ret = apple_typec_phy_sleep_recover(appledwc->usb2_phy); if (ret < 0) { if (!appledwc->sleep_dead) dev_err(appledwc->dev, @@ -1244,7 +1335,7 @@ static bool dwc3_apple_recover(struct dwc3_apple *appledwc) if (ret) dev_err(appledwc->dev, "Failed to stop after the power cycle, err=%d\n", ret); else - dwc3_apple_start_after_sleep(appledwc, DWC3_APPLE_HOST); + dwc3_apple_restart_lost(appledwc); return true; } @@ -1263,7 +1354,7 @@ static bool dwc3_apple_sleep_kept(struct dwc3_apple *appledwc) int ret; appledwc->sleep_in_place = false; - ret = apple_atcphy_sleep_result(appledwc->usb2_phy); + ret = apple_typec_phy_sleep_result(appledwc->usb2_phy); if (ret == -ENODEV) { dwc3_apple_recover(appledwc); return false; @@ -1274,7 +1365,7 @@ static bool dwc3_apple_sleep_kept(struct dwc3_apple *appledwc) dwc3_apple_check_hub_later(appledwc); return true; } - if (!ret || ret == -EBUSY || ret == APPLE_ATCPHY_SLEEP_POWER_LOST) { + if (dwc3_apple_sleep_reads_mark(ret)) { guid = dwc3_readl(dwc, DWC3_GUID); if (guid == DWC3_APPLE_SLEEP_GUID) { dwc3_writel(dwc, DWC3_GUID, appledwc->sleep_guid); @@ -1283,15 +1374,103 @@ static bool dwc3_apple_sleep_kept(struct dwc3_apple *appledwc) return true; } ret = -EIO; + dev_warn(appledwc->dev, + "host lost its state in the sleep (%d, GUID %#x): restarting it\n", + ret, guid); + } else { + /* The mark wasn't read: what the PHY says, or the set-up after it, is why. */ + dev_warn(appledwc->dev, + "host not set up again after the sleep (PHY %d): restarting it\n", ret); } + ret = dwc3_apple_exit(appledwc); + if (ret) + dev_err(appledwc->dev, "Failed to stop after sleep, err=%d\n", ret); + else + dwc3_apple_restart_lost(appledwc); + + return false; +} + +/* + * Whether xHCI is suspended: its controller halted, and nothing of its touching the core. + * An xHCI device without its driver touches nothing either. Takes the xHCI device's lock. + */ +static bool dwc3_apple_xhci_suspended(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + struct usb_hcd *hcd; + bool suspended; + + lockdep_assert_held(&appledwc->lock); + if (!xhci) + return true; + + device_lock(&xhci->dev); + hcd = xhci->dev.driver ? platform_get_drvdata(xhci) : NULL; + suspended = !hcd || !HCD_HW_ACCESSIBLE(hcd); + device_unlock(&xhci->dev); + + return suspended; +} + +/* After a sleep: still suspended, and not by runtime PM, so its resume failed. */ +static bool dwc3_apple_xhci_resume_failed(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + + return xhci && !pm_runtime_suspended(&xhci->dev) && dwc3_apple_xhci_suspended(appledwc); +} + +/* + * A Type-C port's host is kept through this sleep, by its PHY (T6040): see the comment at the + * top. From here to dwc3_apple_sleep_restart_end() the glue's device is an ordinary one for + * the PM core, which then runs dwc3_apple_suspend() after xHCI's suspend, lets the power + * domain go off and orders xHCI's resume after the DARTs' and the PHY's. xHCI has to halt + * for it in any state the sleep ends up in, so it can't be kept running for a USB wakeup. + */ +static void dwc3_apple_sleep_restart_begin(struct dwc3_apple *appledwc) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + + lockdep_assert_held(&appledwc->lock); + appledwc->sleep_state = appledwc->state; + appledwc->sleep_xhci_wakeup = xhci && device_may_wakeup(&xhci->dev); + if (appledwc->sleep_xhci_wakeup) + device_set_wakeup_enable(&xhci->dev, false); + dev_pm_syscore_device(appledwc->dev, false); +} + +/* + * The end of such a sleep, with @kept from dwc3_apple_sleep_kept(), which has restarted a + * host that was lost. A host that was kept is the glue's to keep powered again. If xHCI + * didn't resume on it, or the port controller asked for another role meanwhile, it stops + * now, and what was asked for starts. Returns whether the host is still the one kept. + */ +static bool dwc3_apple_sleep_restart_end(struct dwc3_apple *appledwc, bool kept) +{ + struct platform_device *xhci = appledwc->dwc.xhci; + enum dwc3_apple_state state = appledwc->sleep_state; + int ret; - dev_warn(appledwc->dev, "host lost its state in the sleep (%d, GUID %#x): restarting it\n", - ret, guid); + lockdep_assert_held(&appledwc->lock); + if (!kept || appledwc->state != DWC3_APPLE_HOST) + return false; + + dev_pm_syscore_device(appledwc->dev, true); + if (xhci && appledwc->sleep_xhci_wakeup) + device_set_wakeup_enable(&xhci->dev, true); + if (state == DWC3_APPLE_HOST && !dwc3_apple_xhci_resume_failed(appledwc)) + return true; + + if (state == DWC3_APPLE_HOST) + dev_warn(appledwc->dev, "xHCI did not resume after the sleep: restarting the host\n"); + else + dev_info(appledwc->dev, "role changed during the sleep: stopping the host kept\n"); ret = dwc3_apple_exit(appledwc); if (ret) dev_err(appledwc->dev, "Failed to stop after sleep, err=%d\n", ret); else - dwc3_apple_start_after_sleep(appledwc, DWC3_APPLE_HOST); + dwc3_apple_restart_lost(appledwc); return false; } @@ -1322,11 +1501,22 @@ static int dwc3_apple_pm_notify_locked(struct dwc3_apple *appledwc, unsigned lon *kept = true; break; } + /* A Type-C port's role changed while its PHY was asked: stopped already. */ + if (in_place && appledwc->state != DWC3_APPLE_HOST) { + in_place = false; + *finish = true; + *kept = true; + } /* xHCI suspends and resumes as usual, and the wakeup setting stays. */ if (in_place) { appledwc->sleep_guid = dwc3_readl(&appledwc->dwc, DWC3_GUID); dwc3_writel(&appledwc->dwc, DWC3_GUID, DWC3_APPLE_SLEEP_GUID); appledwc->sleep_in_place = true; + /* On a Type-C port xHCI's wakeup is off for it: nothing there can wake. */ + if (appledwc->sleep_restart) { + dwc3_apple_sleep_restart_begin(appledwc); + break; + } /* For the port controllers' USB wake. Changes after this go unseen. */ apple_sleep_usb_set_wake_ports(dwc3_apple_hub_wake_ports(appledwc)); break; @@ -1344,6 +1534,8 @@ static int dwc3_apple_pm_notify_locked(struct dwc3_apple *appledwc, unsigned lon if (appledwc->sleep_in_place) { apple_sleep_usb_set_wake_ports(0); *kept = dwc3_apple_sleep_kept(appledwc); + if (appledwc->sleep_restart) + *kept = dwc3_apple_sleep_restart_end(appledwc, *kept); *finish = true; break; } @@ -1379,14 +1571,17 @@ static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, * check takes the glue's lock, and has nothing to look at until the sleep is over. */ if (action == PM_SUSPEND_PREPARE) { - apple_atcphy_sleep_finish_flush(appledwc->usb2_phy); + apple_typec_phy_sleep_finish_flush(appledwc->usb2_phy); cancel_delayed_work_sync(&appledwc->hub_check); } if (action == PM_SUSPEND_PREPARE && !pm_suspend_default_s2idle()) { scoped_guard(mutex, &appledwc->lock) - in_place = appledwc->fixed_hub && appledwc->state == DWC3_APPLE_HOST && - !appledwc->shutting_down && !appledwc->sleep_dead; - in_place = in_place && !apple_atcphy_sleep_prepare(appledwc->usb2_phy); + in_place = dwc3_apple_sleep_may_keep(appledwc->fixed_hub, + appledwc->sleep_restart, + appledwc->state, + appledwc->shutting_down, + appledwc->sleep_dead); + in_place = in_place && !apple_typec_phy_sleep_prepare(appledwc->usb2_phy); } scoped_guard(mutex, &appledwc->lock) @@ -1397,13 +1592,56 @@ static int dwc3_apple_pm_notify(struct notifier_block *nb, unsigned long action, * from the PHY's work, which an unbind of this device doesn't wait for under its lock. */ if (finish && action == PM_POST_SUSPEND) - apple_atcphy_sleep_finish_async(appledwc->usb2_phy, kept); + apple_typec_phy_sleep_finish_async(appledwc->usb2_phy, kept); else if (finish) - apple_atcphy_sleep_finish(appledwc->usb2_phy, kept); + apple_typec_phy_sleep_finish(appledwc->usb2_phy, kept); return ret; } +/* + * Only for a Type-C port's host kept through a sleep (T6040, see the comment at the top): + * otherwise the core is either down or the device a syscore one, which the PM core skips. + * xHCI, the child, has suspended: its controller is halted and its state saved, for what + * that's worth. The PHY now takes USB3 and USB2 down and the core into reset, in the order + * of dwc3_apple_core_stop(), and nothing touches the core until dwc3_apple_pre_host_resume(). + * This may sleep, for the PHY's coprocessor, which is why it isn't a noirq callback. + * + * The power domain then goes off in the noirq phase like a stopped controller's, also with a + * USB device below that may wake the system: no wakeup of a powered-down SoC comes through + * the controller. An error refuses the sleep, with the PHY and the core as they were. + */ +static int dwc3_apple_suspend(struct device *dev) +{ + struct dwc3_apple *appledwc = to_dwc3_apple(dev_get_drvdata(dev)); + struct dwc3 *dwc = &appledwc->dwc; + int ret; + + guard(mutex)(&appledwc->lock); + if (!appledwc->sleep_in_place || !appledwc->sleep_restart) + return 0; + + if (!dwc3_apple_xhci_suspended(appledwc)) { + dev_err(dev, "xHCI still runs: refusing the sleep\n"); + return -EBUSY; + } + /* The PHY hands the PIPE back in P3, as it took it over */ + dwc->dis_u3_susphy_quirk = appledwc->dt_dis_u3_susphy; + dwc3_enable_susphy(dwc, true); + ret = apple_typec_phy_sleep_enter(appledwc->usb2_phy); + if (ret) { + dev_err(dev, "PHY did not take the host through the sleep: %d, refusing it\n", ret); + return ret; + } + device_set_out_band_wakeup(dev); + + return 0; +} + +static const struct dev_pm_ops dwc3_apple_pm_ops = { + .suspend = pm_sleep_ptr(dwc3_apple_suspend), +}; + static void dwc3_apple_pm_notifier_remove(void *data) { struct dwc3_apple *appledwc = data; @@ -1437,12 +1675,19 @@ static int dwc3_apple_probe(struct platform_device *pdev) appledwc->dev = &pdev->dev; appledwc->fixed_hub = dwc3_apple_has_fixed_hub(dev, &appledwc->hub_port); appledwc->host_only = dwc3_apple_host_only(dev); + appledwc->sleep_restart = !appledwc->fixed_hub && + of_device_is_compatible(dev->of_node, "apple,t6040-dwc3"); appledwc->dt_dis_u3_susphy = device_property_read_bool(dev, "snps,dis_u3_susphy_quirk"); appledwc->sleep_ddr = device_property_read_bool(dev, "apple,sleep-ddr"); /* The PHY driver and the DARTs follow runtime PM on these SoCs. */ appledwc->idle_rpm = of_device_is_compatible(dev->of_node, "apple,t6040-dwc3") || of_device_is_compatible(dev->of_node, "apple,t8132-dwc3"); mutex_init(&appledwc->lock); + /* + * Its suspend takes 70-120 ms per controller, and its xHCI child, which is asynchronous, + * can't start to resume before the PM core has reached this device in its list. + */ + device_enable_async_suspend(dev); /* A never-connected port has no generic core probe to publish this. */ appledwc->dwc.dev = dev; platform_set_drvdata(pdev, &appledwc->dwc); @@ -1612,6 +1857,7 @@ static struct platform_driver dwc3_apple_driver = { .driver = { .name = "dwc3-apple", .of_match_table = dwc3_apple_of_match, + .pm = pm_ptr(&dwc3_apple_pm_ops), }, }; diff --git a/drivers/usb/typec/bus.c b/drivers/usb/typec/bus.c index e84b134a33813..ceae6ab6584e1 100644 --- a/drivers/usb/typec/bus.c +++ b/drivers/usb/typec/bus.c @@ -535,10 +535,16 @@ static int typec_probe(struct device *dev) } ret = drv->probe(adev); - if (ret) + if (ret) { typec_altmode_remove_links(altmode); + return ret; + } + + /* A mode that was activated before this driver bound pins it now. */ + if (adev->active && !altmode->owner_held) + altmode->owner_held = try_module_get(dev->driver->owner); - return ret; + return 0; } static void typec_remove(struct device *dev) diff --git a/drivers/usb/typec/bus.h b/drivers/usb/typec/bus.h index 7df5deb1dd3a5..bae9885ab68d8 100644 --- a/drivers/usb/typec/bus.h +++ b/drivers/usb/typec/bus.h @@ -22,6 +22,7 @@ struct altmode { const struct attribute_group *groups[2]; struct altmode *partner; + bool owner_held; /* holds the driver's module */ struct altmode *plug[2]; }; diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index 71cf604e25bc4..519637e6e09b2 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -310,11 +310,24 @@ void typec_altmode_update_active(struct typec_altmode *adev, bool active) if (adev->active == active) return; + /* + * The reference pins the driver's module while the mode is active. Only + * a driver that is bound can be pinned: a mode activated before its + * driver bound gets its reference in typec_probe(), and only a taken + * reference is put back. + */ if (!is_typec_port(adev->dev.parent) && adev->dev.driver) { - if (!active) - module_put(adev->dev.driver->owner); - else - WARN_ON(!try_module_get(adev->dev.driver->owner)); + struct altmode *alt = to_altmode(adev); + + if (!active) { + if (alt->owner_held) { + module_put(adev->dev.driver->owner); + alt->owner_held = false; + } + } else if (!alt->owner_held) { + alt->owner_held = try_module_get(adev->dev.driver->owner); + WARN_ON(!alt->owner_held); + } } adev->active = active; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 538959833e4ee..a7ff2155365d6 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -3306,6 +3306,41 @@ static int cd321x_reset_inherited_connection(struct tps6598x *tps, u32 *status) return 0; } +/* + * The controller's system power state S0, and application firmware running: the patch is + * loaded if the controller waits for one (@patched, if given, says so). What a probe and a + * resume from disk both start from. + * + * Return: the controller's mode after the check (0 once a patch was loaded), or a negative + * error code. + */ +static int tipd_start_application(struct tps6598x *tps, bool *patched) +{ + int ret; + + if (patched) + *patched = false; + if (tps->data->switch_power_state) { + ret = tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); + if (ret) + return ret; + } + + ret = tps6598x_check_mode(tps); + if (ret < 0) + return ret; + + if (ret == TPS_MODE_PTCH) { + if (patched) + *patched = true; + ret = tps->data->init(tps); + if (ret) + return ret; + } + + return ret; +} + int tipd_init(struct tps6598x *tps) { struct fwnode_handle *fwnode; @@ -3332,24 +3367,10 @@ int tipd_init(struct tps6598x *tps) } } - if (tps->data->switch_power_state) { - ret = tps->data->switch_power_state(tps, TPS_SYSTEM_POWER_STATE_S0); - if (ret) - return ret; - } - - /* Make sure the controller has application firmware running */ - ret = tps6598x_check_mode(tps); + ret = tipd_start_application(tps, &patch_loaded); if (ret < 0) return ret; - if (ret == TPS_MODE_PTCH) { - patch_loaded = true; - ret = tps->data->init(tps); - if (ret) - return ret; - } - /* * A controller found in "DBMa" is used as it is. Nothing below issues a * mode-changing 4CC command (the S0 power state switch above only does @@ -3630,6 +3651,24 @@ int tipd_suspend(struct tps6598x *tps) } EXPORT_SYMBOL_GPL(tipd_suspend); +/* Give back what tipd_suspend() took: the interrupt, the reset line, the poll. */ +static void tipd_resume_events(struct tps6598x *tps) +{ + if (tps->wakeup) { + if (!tps->sleep_event) { + disable_irq_wake(tps->irq); + enable_irq(tps->irq); + } + } else if (tps->reset) { + gpiod_set_value_cansleep(tps->reset, 0); + msleep(TPS_SETUP_MS); + } + + if (!tps->irq) + queue_delayed_work(system_power_efficient_wq, &tps->wq_poll, + msecs_to_jiffies(POLL_INTERVAL)); +} + int tipd_resume(struct tps6598x *tps) { int ret; @@ -3647,24 +3686,61 @@ int tipd_resume(struct tps6598x *tps) return ret; } - if (tps->wakeup) { - if (!tps->sleep_event) { - disable_irq_wake(tps->irq); - enable_irq(tps->irq); - } - } else if (tps->reset) { - gpiod_set_value_cansleep(tps->reset, 0); - msleep(TPS_SETUP_MS); - } - - if (!tps->irq) - queue_delayed_work(system_power_efficient_wq, &tps->wq_poll, - msecs_to_jiffies(POLL_INTERVAL)); + tipd_resume_events(tps); return 0; } EXPORT_SYMBOL_GPL(tipd_resume); +/* + * Bring a controller that went through a cold boot (a resume from disk) back + * to the state its probe leaves it in, without registering anything again: + * system power state S0, application firmware running (patch loaded if it + * waits for one) and the interrupt mask the driver uses. The image was made + * after tipd_suspend(), so what that took is given back as in tipd_resume(). + * The port state is read afresh by the revalidation that follows. + */ +int tipd_reinit_after_cold_boot(struct tps6598x *tps) +{ + int ret; + + ret = tipd_start_application(tps, NULL); + if (ret < 0) + return ret; + ret = tps6598x_write64(tps, TPS_REG_INT_MASK1, cd321x_irq_mask(tps)); + if (ret) + return ret; + + tipd_resume_events(tps); + + return 0; +} +EXPORT_SYMBOL_GPL(tipd_reinit_after_cold_boot); + +/* + * Look the connector's USB role switch up again. After a resume from disk the + * USB controller that provides it was bound again, which registered a new + * switch; the old one is unregistered and only refuses requests. + */ +int tipd_refresh_role_switch(struct tps6598x *tps) +{ + struct fwnode_handle *fwnode; + struct usb_role_switch *sw, *old; + + fwnode = device_get_named_child_node(tps->dev, "connector"); + sw = fwnode_usb_role_switch_get(fwnode); + fwnode_handle_put(fwnode); + if (IS_ERR(sw)) + return PTR_ERR(sw); + scoped_guard(mutex, &tps->lock) { + old = tps->role_sw; + tps->role_sw = sw; + } + usb_role_switch_put(old); + return 0; +} +EXPORT_SYMBOL_GPL(tipd_refresh_role_switch); + /* Freezable cable work has drained before device PM calls prepare. */ int tipd_prepare(struct device *dev) { diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index ddd28b8d94e46..cb730e11e614a 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -51,7 +52,7 @@ static int regmap_sn201202x_select_reg(struct spmi_device *sdev, u8 reg) if (err) return err; - if (sn->polled || sn->select_polled) { + if (sn->polled || sn->select_polled || READ_ONCE(sn->cold_polled)) { int poll_err; if (sn->atomic) @@ -66,8 +67,13 @@ static int regmap_sn201202x_select_reg(struct spmi_device *sdev, u8 reg) return err ?: poll_err; } - if (!wait_for_completion_timeout(&sn->select_completion, msecs_to_jiffies(100))) - return -ETIMEDOUT; + if (!wait_for_completion_timeout(&sn->select_completion, msecs_to_jiffies(100))) { + /* The selection may have completed without its interrupt. */ + err = spmi_register_read(sdev, 0, &val); + if (!err && val == reg) + return 0; + return err ?: -ETIMEDOUT; + } while (--attempts) { err = spmi_register_read(sdev, 0, &val); @@ -504,6 +510,10 @@ static const struct of_device_id sn201202x_of_match[] = { /* * In system sleep the wake interrupt is suspended, so the controller is * polled as without interrupts. + * + * A controller that is awake answers a WAKEUP with no interrupt. One whose + * wake interrupt has come since it was sent to sleep woke by itself: it is + * checked at once, as when polled, instead of waited for. */ static int sn201202x_wake_poll(struct sn201202x *sn, bool poll) { @@ -514,6 +524,8 @@ static int sn201202x_wake_poll(struct sn201202x *sn, bool poll) u8 size; int ret; + if (completion_done(&sn->wake_completion)) + poll = true; reinit_completion(&sn->wake_completion); ret = spmi_command_wakeup(sn->sdev); if (ret) @@ -579,6 +591,8 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_runtime_suspend(struct device *dev return -EBUSY; reinit_completion(&sn->sleep_completion); + /* A wake interrupt from here on is its own, see sn201202x_wake_poll() */ + reinit_completion(&sn->wake_completion); if (spmi_command_sleep(sn->sdev)) goto retry; if (!sn->polled && @@ -711,6 +725,60 @@ static bool sn201202x_sleep_event(struct tps6598x *tps) return true; } +/* + * A port restored from disk is revalidated once its USB role switch is there. + * The USB controller that provides it does not come through a resume from disk + * bound, and its new probe registers a new switch: the one looked up before + * the image was made is unregistered and only refuses requests. Take the new + * one and let the port read its cable afresh, without taking the port, or a + * display on it, down. + * + * Return: 0 once the port revalidates, or why the switch could not be taken. + */ +static int sn201202x_roles_finish(struct sn201202x *sn) +{ + struct tps6598x *tps = &sn->cd.tps; + int ret; + + ret = tipd_refresh_role_switch(tps); + if (ret) + return ret; + scoped_guard(mutex, &tps->lock) { + if (!sn->roles_pending) + return 0; + WRITE_ONCE(sn->roles_pending, false); + /* Refresh the detector snapshot before cable revalidation is thawed. */ + sn201202x_ldcm_refresh(tps); + } + tipd_complete(tps->dev); + dev_dbg(tps->dev, "resume from disk: USB role switch looked up again, port revalidates\n"); + return 0; +} + +static void sn201202x_role_work(struct work_struct *work) +{ + struct sn201202x *sn = container_of(work, struct sn201202x, role_work); + + if (READ_ONCE(sn->roles_pending)) + sn201202x_roles_finish(sn); +} + +/* + * A USB controller has bound, and its probe registers its role switch: a port + * that waits for one after a resume from disk looks again. + */ +static int sn201202x_role_notify(struct notifier_block *nb, unsigned long action, void *data) +{ + struct sn201202x *sn = container_of(nb, struct sn201202x, role_nb); + + if (action != BUS_NOTIFY_BOUND_DRIVER || !READ_ONCE(sn->roles_pending) || + !device_property_present(data, "usb-role-switch")) + return NOTIFY_DONE; + + queue_work(system_freezable_wq, &sn->role_work); + return NOTIFY_OK; +} + static int sn201202x_probe(struct spmi_device *device) { const struct of_device_id *match; @@ -756,6 +824,7 @@ static int sn201202x_probe(struct spmi_device *device) if (!of_property_read_bool(device->dev.of_node, "apple,no-liquid-detection")) tps->contaminant_ops = &sn201202x_contaminant_ops; INIT_DELAYED_WORK(&sn->ldcm_work, sn201202x_ldcm_work); + INIT_WORK(&sn->role_work, sn201202x_role_work); init_completion(&sn->select_completion); init_completion(&sn->sleep_completion); @@ -828,6 +897,12 @@ static int sn201202x_probe(struct spmi_device *device) tps->data->irq_handler(tps->irq, tps); sn201202x_idle_sleep_start(tps); + + sn->role_nb.notifier_call = sn201202x_role_notify; + ret = bus_register_notifier(&platform_bus_type, &sn->role_nb); + if (ret) + dev_warn(&device->dev, "cannot watch for the USB role switch: %d\n", ret); + sn->role_watch = !ret; return 0; } @@ -836,6 +911,12 @@ static void sn201202x_remove(struct spmi_device *device) struct tps6598x *tps = spmi_device_get_drvdata(device); struct sn201202x *sn = tps_to_sn(tps); + if (sn->role_watch) + bus_unregister_notifier(&platform_bus_type, &sn->role_nb); + disable_work_sync(&sn->role_work); + /* The hold freeze took on one the restore left alone */ + if (sn->cold_restored) + pm_runtime_put_noidle(tps->dev); sn201202x_idle_sleep_stop(tps); tipd_remove(tps); spmi_command_sleep(sn->sdev); @@ -847,10 +928,16 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_resume(struct device *dev) struct sn201202x *sn = tps_to_sn(tps); int err; + /* Its suspend left it alone too: interrupt masked, still held awake */ + if (sn->cold_restored) + return 0; + if (READ_ONCE(sn->sleep_watch)) { /* Waits for a running sleep_event() */ disable_irq(tps->irq); WRITE_ONCE(sn->sleep_watch, false); + /* The watch woke it and sent it to sleep again: not its own wake */ + reinit_completion(&sn->wake_completion); } /* One that slept on from idle sleep wakes through runtime PM */ if (pm_runtime_status_suspended(dev)) { @@ -880,6 +967,10 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_suspend(struct device *dev) bool watch; int err, ret; + /* A controller a resume from disk left dead waits for its new probe. */ + if (sn->cold_restored) + return 0; + /* * A sleeping controller cannot be read, so its events must not be * handled until it is awake again. An interrupt arriving meanwhile is @@ -917,6 +1008,8 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_suspend(struct device *dev) /* Registered before xHCI, so its root port has suspended by now. */ tipd_usb_wake(tps, true); reinit_completion(&sn->sleep_completion); + /* A wake interrupt from here on is its own, see sn201202x_wake_poll() */ + reinit_completion(&sn->wake_completion); err = spmi_command_sleep(sn->sdev); if (err) goto out_resume; @@ -946,12 +1039,88 @@ VISIBLE_IF_KUNIT int __maybe_unused sn201202x_suspend(struct device *dev) } EXPORT_SYMBOL_IF_KUNIT(sn201202x_suspend); +/* + * A resume from hibernation is a cold boot: the boot loader and the kernel + * that loaded the image have taken the controller through their own bring-up, + * so the state saved in the image (its sleep, the latched events) no longer + * exists. Wake the controller and set it up again in place, as its probe + * would; its port is revalidated once the connector's USB role switch is + * there, see sn201202x_roles_finish(). A controller that does not take this + * is left alone, with its event interrupt still disabled, for a new probe. + */ +static int sn201202x_restore(struct device *dev) +{ + struct tps6598x *tps = dev_get_drvdata(dev); + struct sn201202x *sn = tps_to_sn(tps); + int err; + const char *step; + + /* + * An earlier restore left it for a new probe: freeze took no hold + * and left its interrupt alone, so there is nothing to give back. + */ + if (sn->cold_restored) + return 0; + + /* + * The cold boot left the controller awake, so a wakeup command raises + * no wake interrupt: it is woken polled. Until it is set up again its + * register selections are polled as well, as on a controller without + * interrupts. + */ + WRITE_ONCE(sn->cold_polled, true); + step = "wake"; + err = sn201202x_wake_poll(sn, true); + if (!err) { + step = "set-up"; + err = tipd_reinit_after_cold_boot(tps); + } + if (err) { + /* + * Interrupt still masked and held awake from freeze: nothing + * reads a dead chip, and it does not sleep while idle. + */ + sn->cold_restored = true; + dev_warn(dev, "resume from disk: restore failed at %s (%d), controller left for a new probe\n", + step, err); + return 0; + } + /* The cold boot reset the controller, which also removed its mitigation. */ + scoped_guard(mutex, &tps->lock) + sn201202x_ldcm_invalidate(tps); + /* Set up as after probe: from here on by its interrupts again. */ + WRITE_ONCE(sn->cold_polled, false); + if (tps->irq) + enable_irq(tps->irq); + /* Give back the hold freeze took, as resume does */ + pm_runtime_put_autosuspend(dev); + scoped_guard(mutex, &tps->lock) { + WRITE_ONCE(sn->roles_pending, true); + sn->hib_in_place = true; + } + dev_dbg(dev, "resume from disk: controller restored, port revalidation waits for the USB role switch\n"); + return 0; +} + +/* + * A port still waiting for its role switch at the next system sleep gives up + * the wait: that sleep's resume revalidates it with whatever switch is there (a + * missing one only refuses). A controller the restore from disk left for a new + * probe is not asked for its connector state. + */ static int sn201202x_prepare(struct device *dev) { struct tps6598x *tps = dev_get_drvdata(dev); struct sn201202x *sn = tps_to_sn(tps); int ret; + scoped_guard(mutex, &tps->lock) { + WRITE_ONCE(sn->roles_pending, false); + sn->hib_in_place = false; + } + if (sn->cold_restored) + return tipd_prepare(dev); + cancel_delayed_work_sync(&sn->ldcm_work); mutex_lock(&tps->lock); ret = sn201202x_ldcm_update(tps); @@ -974,19 +1143,77 @@ static int sn201202x_prepare(struct device *dev) static void sn201202x_complete(struct device *dev) { struct tps6598x *tps = dev_get_drvdata(dev); + struct sn201202x *sn = tps_to_sn(tps); + /* A controller the restore from disk left alone is probed again instead. */ + if (sn->cold_restored) + return; + /* + * A port restored from disk waits for its USB role switch: now, if the + * USB controller is bound, or when sn201202x_role_notify() sees it bind. + */ + if (READ_ONCE(sn->roles_pending)) { + sn201202x_roles_finish(sn); + return; + } /* Refresh the detector snapshot before cable revalidation is thawed. */ scoped_guard(mutex, &tps->lock) sn201202x_ldcm_refresh(tps); tipd_complete(dev); } +/* + * hib_resume: how the last resume from disk left the controller: "none", + * "roles-pending" (restored in place) or "failed" (to be probed again). The + * kernel revalidates a port restored in place by itself; writing "roles" only + * takes note of it, after which the attribute reads "none". + */ +static ssize_t hib_resume_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct tps6598x *tps = dev_get_drvdata(dev); + struct sn201202x *sn = tps_to_sn(tps); + + guard(mutex)(&tps->lock); + return sysfs_emit(buf, "%s\n", sn->cold_restored ? "failed" : + sn->hib_in_place ? "roles-pending" : "none"); +} + +static ssize_t hib_resume_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct tps6598x *tps = dev_get_drvdata(dev); + struct sn201202x *sn = tps_to_sn(tps); + + if (!sysfs_streq(buf, "roles")) + return -EINVAL; + scoped_guard(mutex, &tps->lock) { + if (!sn->hib_in_place) + return -EALREADY; + sn->hib_in_place = false; + } + /* A port that still waits looks again, as this write used to make it. */ + queue_work(system_freezable_wq, &sn->role_work); + return count; +} +static DEVICE_ATTR_RW(hib_resume); + +static struct attribute *sn201202x_attrs[] = { + &dev_attr_hib_resume.attr, + NULL +}; +ATTRIBUTE_GROUPS(sn201202x); + MODULE_DEVICE_TABLE(of, sn201202x_of_match); static const struct dev_pm_ops sn201202x_pm_ops = { .prepare = pm_sleep_ptr(sn201202x_prepare), .complete = pm_sleep_ptr(sn201202x_complete), - SET_SYSTEM_SLEEP_PM_OPS(sn201202x_suspend, sn201202x_resume) + .suspend = pm_sleep_ptr(sn201202x_suspend), + .resume = pm_sleep_ptr(sn201202x_resume), + .freeze = pm_sleep_ptr(sn201202x_suspend), + .thaw = pm_sleep_ptr(sn201202x_resume), + .poweroff = pm_sleep_ptr(sn201202x_suspend), + .restore = pm_sleep_ptr(sn201202x_restore), SET_RUNTIME_PM_OPS(sn201202x_runtime_suspend, sn201202x_runtime_resume, NULL) }; @@ -1000,6 +1227,7 @@ static struct spmi_driver sn201202x_driver = { * port; let the ports of one machine probe at the same time. */ .probe_type = PROBE_PREFER_ASYNCHRONOUS, + .dev_groups = sn201202x_groups, }, .probe = sn201202x_probe, .remove = sn201202x_remove, diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 68d246b7397e1..0195611ab8cde 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -508,6 +508,18 @@ struct sn201202x { bool ldcm_mitigated; /* ldcm_mitigated matches the controller (unknown after a reset) */ bool ldcm_mitigation_known; + /* Restored from disk: left untouched until it is probed again. */ + bool cold_restored; + /* Restored from disk: revalidation waits for the new USB role switch. */ + bool roles_pending; + /* Restored from disk in place, as hib_resume reports it (tps.lock) */ + bool hib_in_place; + /* Looks the role switch up again when a USB controller binds */ + struct notifier_block role_nb; + struct work_struct role_work; + bool role_watch; + /* Restored from disk: register selections are polled until it is set up again. */ + bool cold_polled; }; extern const struct tipd_data *tipd_hw_variants[]; @@ -519,6 +531,8 @@ int tipd_prepare(struct device *dev); void tipd_complete(struct device *dev); int tipd_suspend(struct tps6598x *tps); int tipd_resume(struct tps6598x *tps); +int tipd_refresh_role_switch(struct tps6598x *tps); +int tipd_reinit_after_cold_boot(struct tps6598x *tps); /* Internal to tps6598x-core */ int tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len); diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index 530809696b254..da51c9be40efb 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -120,8 +120,14 @@ struct apple_smc { bool sleep_warned; /* To be told at the noirq suspend, after the debug link's cut. */ bool sleep_warn_late; + /* A sleep was committed, and the wake after it has not been declared a full one. */ + bool wake_silent; + /* This SoC's SMC is told of a sleep and of a full wake through WKTP as well. */ + bool wktp; /* In a noirq phase: replies are polled for. */ bool polled; + /* Registered with the sleep controller, whose last stops can send the sleep commit. */ + bool commit_at_stop; /* MBSW at the last wake: what the SMC had woken the AP for. */ u64 wake_record; /* MBSW's bits as this SoC numbers them, from the probe. */ @@ -299,6 +305,18 @@ static inline int macsmc_hwmon_fans_request(struct device *dev, unsigned int per } #endif +/* + * Request the VDD_CIO rail (SMC key pmVC) for a system sleep: see drivers/mfd/macsmc.c. + */ +#if IS_REACHABLE(CONFIG_MFD_MACSMC) +int apple_smc_request_pmvc(struct apple_smc *smc); +#else +static inline int apple_smc_request_pmvc(struct apple_smc *smc) +{ + return -ENODEV; +} +#endif + static inline int apple_smc_read_flag(struct apple_smc *smc, smc_key key, bool *flag) { u8 val; diff --git a/include/linux/phy/apple-atc.h b/include/linux/phy/apple-atc.h index c81d09de1ce5c..2605d43bc6904 100644 --- a/include/linux/phy/apple-atc.h +++ b/include/linux/phy/apple-atc.h @@ -8,13 +8,21 @@ #include #include - -struct phy; +#include /* In D0 after the sleep, but its always-on side lost power, so xHCI's saved state is gone. */ #define APPLE_ATCPHY_SLEEP_POWER_LOST 1 -#if IS_REACHABLE(CONFIG_PHY_APPLE_ATC) +/* + * The PHY held the controller in reset through the sleep, with its own USB2 and USB3 sides + * down, and has released the reset: the core's registers are gone whether or not the SoC + * lost power, and so is xHCI's saved state. The PHY's own sides come back with + * apple_typec_phy_sleep_power_up(), once the core is set up again. + */ +#define APPLE_ATCPHY_SLEEP_RESET 2 + +extern const struct phy_ops apple_atc_usb2_phy_ops; + /* * For a firmware-driven system sleep, before any device suspends and without the * controller's locks held. 0 means the PHY takes the controller through hibernation after @@ -49,58 +57,158 @@ void apple_atcphy_sleep_finish_async(struct phy *usb2_phy, bool kept); void apple_atcphy_sleep_finish_flush(struct phy *usb2_phy); /* ACIO owns this transaction; @phy is its shared USB3 lane PHY handle. */ +#if IS_REACHABLE(CONFIG_PHY_APPLE_ATC) int apple_atcphy_cio_prepare(struct phy *phy); int apple_atcphy_cio_off(struct phy *phy); int apple_atcphy_cio_on(struct phy *phy); int apple_atcphy_cio_finish(struct phy *phy); #else -static inline int apple_atcphy_sleep_prepare(struct phy *usb2_phy) +static inline int apple_atcphy_cio_prepare(struct phy *phy) { return -EOPNOTSUPP; } -static inline int apple_atcphy_sleep_result(struct phy *usb2_phy) +static inline int apple_atcphy_cio_off(struct phy *phy) { return -EOPNOTSUPP; } -static inline int apple_atcphy_sleep_recover(struct phy *usb2_phy) +static inline int apple_atcphy_cio_on(struct phy *phy) { return -EOPNOTSUPP; } -static inline void apple_atcphy_sleep_finish(struct phy *usb2_phy, bool kept) +static inline int apple_atcphy_cio_finish(struct phy *phy) { + return -EOPNOTSUPP; } +#endif -static inline void apple_atcphy_sleep_finish_async(struct phy *usb2_phy, bool kept) +extern const struct phy_ops apple_t6040_usb2_phy_ops; + +/* + * The T6040 Type-C PHY's side of the same sleep, for a Type-C port in host mode with nothing + * but USB on its lanes. It has no hibernation to take the controller through: it takes its + * USB3 and USB2 sides down and the controller into reset, as a stop of the host does, and + * brings them back the way a start does. That boots the PHY's coprocessor, so none of it + * runs with interrupts off: the controller's glue calls each step from process context. + * Each returns -EOPNOTSUPP for a PHY that isn't this driver's. + * + * apple_t6040_atcphy_sleep_prepare(): before any device suspends. 0 if the port can sleep + * this way now, and the PHY then holds on to that until the finish. + */ +int apple_t6040_atcphy_sleep_prepare(struct phy *usb2_phy); + +/* + * Once xHCI has suspended, and before the controller's power domain may go off: USB3 down + * and its PIPE parked, USB2 off, the controller in reset. -EBUSY, with nothing touched, if + * the port has changed since the prepare; the sleep has to be refused then. + */ +int apple_t6040_atcphy_sleep_enter(struct phy *usb2_phy); + +/* + * After the sleep, with the controller's power domain on again and before anything touches + * its core. 0: the PHY was never taken down (the sleep was abandoned first), and all is as + * it was. APPLE_ATCPHY_SLEEP_RESET: the controller is out of reset again, which the first + * call after the sleep sees to. -ENODEV: the reset didn't release, so the core must not + * be touched. Any other error: the controller is out of reset, but USB2 didn't come back. + */ +int apple_t6040_atcphy_sleep_result(struct phy *usb2_phy); + +/* + * After APPLE_ATCPHY_SLEEP_RESET, once the core is set up again and before its PIPE is + * routed: USB2 on, and USB3 up if it ran before the sleep. May sleep for the coprocessor's + * boot. USB3 that doesn't come up leaves the port at USB2, as at a start of the host, and + * is not an error. An error is USB2's, and leaves the controller out of reset for a restart + * of the host. + */ +int apple_t6040_atcphy_sleep_power_up(struct phy *usb2_phy); + +/* For -ENODEV: tries the reset's release again. APPLE_ATCPHY_SLEEP_RESET, or -ENODEV. */ +int apple_t6040_atcphy_sleep_recover(struct phy *usb2_phy); + +/* + * Ends what the prepare took on. Without @kept, a failure of the PHY's own makes the host + * stop for sleeps until it has started again. + */ +void apple_t6040_atcphy_sleep_finish(struct phy *usb2_phy, bool kept); + +/* Whether @usb2_phy is the USB2 PHY of a driver that is built for this kernel. */ +static inline bool apple_atcphy_is_atc(struct phy *usb2_phy) { + return IS_REACHABLE(CONFIG_PHY_APPLE_ATC) && usb2_phy && + usb2_phy->ops == &apple_atc_usb2_phy_ops; } -static inline void apple_atcphy_sleep_finish_flush(struct phy *usb2_phy) +static inline bool apple_atcphy_is_t6040(struct phy *usb2_phy) { + return IS_REACHABLE(CONFIG_PHY_APPLE_T6040_USB2) && usb2_phy && + usb2_phy->ops == &apple_t6040_usb2_phy_ops; } -static inline int apple_atcphy_cio_prepare(struct phy *phy) +/* + * What the controller's glue calls: the driver that @usb2_phy belongs to, found by the PHY's + * ops, answers. A step that only one of the drivers has is nothing to do for the other's PHYs, + * and for any other PHY. + */ +static inline int apple_typec_phy_sleep_prepare(struct phy *usb2_phy) { + if (apple_atcphy_is_atc(usb2_phy)) + return apple_atcphy_sleep_prepare(usb2_phy); + if (apple_atcphy_is_t6040(usb2_phy)) + return apple_t6040_atcphy_sleep_prepare(usb2_phy); return -EOPNOTSUPP; } -static inline int apple_atcphy_cio_off(struct phy *phy) +static inline int apple_typec_phy_sleep_enter(struct phy *usb2_phy) { - return -EOPNOTSUPP; + return apple_atcphy_is_t6040(usb2_phy) ? apple_t6040_atcphy_sleep_enter(usb2_phy) : 0; } -static inline int apple_atcphy_cio_on(struct phy *phy) +static inline int apple_typec_phy_sleep_result(struct phy *usb2_phy) { + if (apple_atcphy_is_atc(usb2_phy)) + return apple_atcphy_sleep_result(usb2_phy); + if (apple_atcphy_is_t6040(usb2_phy)) + return apple_t6040_atcphy_sleep_result(usb2_phy); return -EOPNOTSUPP; } -static inline int apple_atcphy_cio_finish(struct phy *phy) +static inline int apple_typec_phy_sleep_power_up(struct phy *usb2_phy) { + return apple_atcphy_is_t6040(usb2_phy) ? apple_t6040_atcphy_sleep_power_up(usb2_phy) : 0; +} + +static inline int apple_typec_phy_sleep_recover(struct phy *usb2_phy) +{ + if (apple_atcphy_is_atc(usb2_phy)) + return apple_atcphy_sleep_recover(usb2_phy); + if (apple_atcphy_is_t6040(usb2_phy)) + return apple_t6040_atcphy_sleep_recover(usb2_phy); return -EOPNOTSUPP; } -#endif +static inline void apple_typec_phy_sleep_finish(struct phy *usb2_phy, bool kept) +{ + if (apple_atcphy_is_atc(usb2_phy)) + apple_atcphy_sleep_finish(usb2_phy, kept); + else if (apple_atcphy_is_t6040(usb2_phy)) + apple_t6040_atcphy_sleep_finish(usb2_phy, kept); +} + +/* The T6040 PHY's finish has nothing to wait for, so it ends at once. */ +static inline void apple_typec_phy_sleep_finish_async(struct phy *usb2_phy, bool kept) +{ + if (apple_atcphy_is_atc(usb2_phy)) + apple_atcphy_sleep_finish_async(usb2_phy, kept); + else if (apple_atcphy_is_t6040(usb2_phy)) + apple_t6040_atcphy_sleep_finish(usb2_phy, kept); +} + +static inline void apple_typec_phy_sleep_finish_flush(struct phy *usb2_phy) +{ + if (apple_atcphy_is_atc(usb2_phy)) + apple_atcphy_sleep_finish_flush(usb2_phy); +} #endif diff --git a/include/linux/phy/apple-t6040-cio.h b/include/linux/phy/apple-t6040-cio.h new file mode 100644 index 0000000000000..d041a3a3db9f8 --- /dev/null +++ b/include/linux/phy/apple-t6040-cio.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple T6040 Type-C PHY: the USB4 host router's view of the PHY's CIO + * (USB4/Thunderbolt) mode. The Type-C controller puts the PHY into CIO + * through the mux before it hands the cable to the router, and takes it out + * after the router has stopped. The router uses these calls to check that + * order and to tell the PHY whether it runs, which the PHY needs before it + * takes CIO down for system sleep. @phy is the PHY the router's "phys" + * property names, got with devm_phy_get(): the PHY core links the router to it + * as its consumer, so the PHY is bound before the router probes. + */ +#ifndef __LINUX_PHY_APPLE_T6040_CIO_H +#define __LINUX_PHY_APPLE_T6040_CIO_H + +#include +#include +#include + +struct phy; + +#if IS_REACHABLE(CONFIG_PHY_APPLE_T6040_USB2) +/* + * 1: CIO up and tuned by the PHY firmware; 0: not in CIO; -ENODEV: @phy is + * not a T6040 Type-C PHY. + */ +int apple_t6040_cio_state(struct phy *phy); + +/* + * Before the router starts: marks it running on the PHY's CIO link. + * -EAGAIN unless CIO is up; -ENODEV as above. + */ +int apple_t6040_cio_router_started(struct phy *phy); + +/* + * After every router stop, once ACIO is powered down (unplug, error, and the + * stop for system sleep, which must come before any device's .suspend, + * .freeze or .poweroff). Undoes apple_t6040_cio_router_started(). + */ +void apple_t6040_cio_router_stopped(struct phy *phy); + +/* + * The router carries (@on) or no longer carries USB3 for the port's USB + * controller: the controller's PIPE follows. -EAGAIN when @on and the router + * is not marked running. + */ +int apple_t6040_cio_usb3_tunnel(struct phy *phy, bool on); +#else +static inline int apple_t6040_cio_state(struct phy *phy) +{ + return -ENODEV; +} + +static inline int apple_t6040_cio_router_started(struct phy *phy) +{ + return -ENODEV; +} + +static inline void apple_t6040_cio_router_stopped(struct phy *phy) +{ +} + +static inline int apple_t6040_cio_usb3_tunnel(struct phy *phy, bool on) +{ + return -ENODEV; +} +#endif + +#endif diff --git a/include/linux/soc/apple/aura-hibernate.h b/include/linux/soc/apple/aura-hibernate.h new file mode 100644 index 0000000000000..729ced7b3c4cb --- /dev/null +++ b/include/linux/soc/apple/aura-hibernate.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __LINUX_SOC_APPLE_AURA_HIBERNATE_H +#define __LINUX_SOC_APPLE_AURA_HIBERNATE_H + +#include +#include + +struct device; +struct device_node; + +/* A firmware memory-region of a device as this boot's device tree gives it. */ +struct aura_fw_region { + u64 phys; + u64 iova; + u64 size; +}; + +/* + * Declared unconditionally so that Rust drivers can bind to them; without + * CONFIG_APPLE_AURA_HIBERNATE they must not call them. + */ +int aura_hibernate_boot_fw_regions(struct device_node *np, struct aura_fw_region *out, + unsigned int max); +int aura_hibernate_reprobe_after_thaw(struct device *dev); + +#ifdef CONFIG_APPLE_AURA_HIBERNATE +int aura_hibernate_boot_chosen_u64(const char *name, u64 *val); +#else +static inline int aura_hibernate_boot_chosen_u64(const char *name, u64 *val) { return -ENODEV; } +#endif + +#endif diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 034a74b852ebc..6118de23d143d 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -59,6 +59,12 @@ int apple_dcp_tb_dp_event(struct device_node *connector_np, unsigned int dpin, */ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate); +/* + * T6040 ATC PHY: the same for DP IN adapter @dpin (0 or 1), a symbol of its own because + * phy-apple-atc and phy-apple-t6040-usb2 can both be built. + */ +int apple_t6040_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate); + /* T6030's external DPTX port 0 sources feed DP IN0/1 through PCLK1/2. */ static inline int apple_dp_tunnel_t6030_pclk(unsigned int source, unsigned int dpin) { diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 9cc33b48e334a..c42c68da4d6f5 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -21,6 +21,7 @@ int apple_pmp_report_ane_perf(const struct device_node *np, struct device *owner, unsigned int level, bool *held); int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms); int apple_pmp_report_wait_running(struct device_node *report_np, unsigned int timeout_ms); +int apple_pmp_report_wait_handed(unsigned int timeout_ms); /* Caller holds the PMP owner's vote lock and keeps the supplier bound. */ int apple_pmp_report_check_t6030_gfx(struct device *supplier); int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); @@ -113,6 +114,11 @@ static inline int apple_pmp_report_wait_running(struct device_node *report_np, return -ENODEV; } +static inline int apple_pmp_report_wait_handed(unsigned int timeout_ms) +{ + return 0; +} + static inline int apple_pmp_report_check_t6030_gfx(struct device *supplier) { return -ENODEV; diff --git a/include/linux/soc/apple/pmp.h b/include/linux/soc/apple/pmp.h index 8663ca195febe..59e9c05ff1e68 100644 --- a/include/linux/soc/apple/pmp.h +++ b/include/linux/soc/apple/pmp.h @@ -49,4 +49,6 @@ struct apple_pmp_commit_stop { }; int apple_pmp_sleep_prepare(const void *data, struct apple_pmp_commit_stop *stop); +/* Rebuild the firmware session after a resume from disk (PM_POST_HIBERNATION). */ +int apple_pmp_restore(const void *data); #endif diff --git a/include/linux/soc/apple/rtkit-helper.h b/include/linux/soc/apple/rtkit-helper.h index 190caf432e098..4c712936e6c15 100644 --- a/include/linux/soc/apple/rtkit-helper.h +++ b/include/linux/soc/apple/rtkit-helper.h @@ -9,4 +9,20 @@ struct device; /* The caller holds a device link to the bound helper supplier. */ bool apple_rtkit_helper_preserves_input(struct device *dev); +/* + * After a resume from disk: the helper took over, in its .restore callback, + * the session that the boot kernel started and left running. Valid from the + * helper's .restore until its next .freeze. + */ +bool apple_rtkit_helper_readopted(struct device *dev); + +/* + * After a resume from disk: the session this helper had started itself was + * found with its co-processor halted (the boot loader stopped it and the boot + * kernel did not start it). Nothing answers until the helper is probed again, + * which starts the co-processor. Valid from the helper's .restore until its + * next .freeze. + */ +bool apple_rtkit_helper_halted_underneath(struct device *dev); + #endif diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 244e60bbc6989..44514f2c32eaa 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -264,6 +265,13 @@ int apple_rtkit_poweroff(struct apple_rtkit *rtk); */ int apple_rtkit_cold_restart(struct apple_rtkit *rtk, int (*start)(void *), void *ctx); +/* + * Returned by apple_rtkit_idle() and apple_rtkit_idle_with() when the AP side is idle + * and the IOP acknowledged a power state other than idle: it is asleep with its memory + * kept. No other error carries this value. + */ +#define APPLE_RTKIT_IDLE_RETAINED (-EALREADY) + /* * Put the co-processor into idle mode */ @@ -310,6 +318,16 @@ int apple_rtkit_ap_power(struct apple_rtkit *rtk, bool on); /* The same, for a noirq phase: device interrupts are off, so it polls. */ int apple_rtkit_ap_power_noirq(struct apple_rtkit *rtk, bool on); +/* + * The idle notice with interrupts off and one CPU up, where nothing can sleep and the + * receive work doesn't run: the mailbox is polled a bounded number of times, with no + * clock read, and the acknowledgment is taken in the poll. A message on one of the + * client's own endpoints still goes to recv_message_early; one it leaves waits for the + * receive work. @us is about how long the acknowledgment took. The wake's notice is + * apple_rtkit_ap_power() or apple_rtkit_ap_power_noirq(), as after any other. + */ +int apple_rtkit_ap_idle_polled(struct apple_rtkit *rtk, unsigned int *us); + /* * Stop a co-processor for a system sleep that may remove its power: idle, wait * for the status that shows it has finished, then clear the run bit. On an error @@ -442,4 +460,60 @@ void apple_rtkit_flush_rx(struct apple_rtkit *rtk); */ bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep); +/* + * A shared buffer of a running session, as the co-processor addresses it. + * @kind is one of APPLE_RTKIT_BUF_*. + */ +#define APPLE_RTKIT_BUF_IOREPORT 0 +#define APPLE_RTKIT_BUF_COREANALYTICS 1 +#define APPLE_RTKIT_BUF_CRASHLOG 2 +#define APPLE_RTKIT_BUF_OSLOG 3 +#define APPLE_RTKIT_BUF_KDEBUG0 4 +#define APPLE_RTKIT_BUF_KDEBUG1 5 +#define APPLE_RTKIT_BUF_PMP_AUX 6 +#define APPLE_RTKIT_BUF_SYSLOG 7 +#define APPLE_RTKIT_BUF_COUNT 8 + +struct apple_rtkit_buffer_desc { + u32 kind; + u32 is_mapped; + u64 iova; + u64 size; +}; + +/* + * Describe the shared buffers the co-processor currently holds, in a fixed + * order. Returns the number of entries written, or -ENOSPC if @max is too + * small. + */ +int apple_rtkit_describe_buffers(struct apple_rtkit *rtk, + struct apple_rtkit_buffer_desc *desc, + unsigned int max); + +/* + * Take over, in place, a firmware instance that another kernel started and + * left running with its shared buffers at the IOVAs this session already + * holds (the caller must have checked that with apple_rtkit_describe_buffers() + * against the other kernel's description). Only host-side transient state is + * reset; no message is sent to the co-processor. + */ +int apple_rtkit_readopt(struct apple_rtkit *rtk); + +/* + * Resume from disk. The kernel that loads a hibernation image (the boot + * kernel) can leave a few bytes for the kernel restored from that image in + * memory that the image does not overwrite. Only available with RTKit + * built in; elsewhere put() fails and take() finds nothing. + * + * apple_rtkit_hib_restoring(): true in the boot kernel between + * PM_RESTORE_PREPARE and the jump into the image (or PM_POST_RESTORE), + * i.e. while its devices are quiesced for the image, not frozen for one. + */ +#define APPLE_RTKIT_HIB_HANDOFF_MAX 256 + +bool apple_rtkit_hib_restoring(void); +int apple_rtkit_hib_handoff_put(const char *tag, const void *data, size_t len); +int apple_rtkit_hib_handoff_take(const char *tag, void *data, size_t len); +void apple_rtkit_hib_handoff_drop(const char *tag); + #endif /* _LINUX_APPLE_RTKIT_H_ */ diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h index 07e03087cbc3d..ddd15c742e2de 100644 --- a/include/linux/soc/apple/sleep.h +++ b/include/linux/soc/apple/sleep.h @@ -9,11 +9,22 @@ #include #include #include +#include #include struct device; struct device_node; +/* + * The coprocessor and enclave registry, shared by the sleep controllers + * (drivers/soc/apple/sleep-registry.c). The controller bound on this SoC + * answers apple_sleep_commit_stops_coprocs() and runs the last stops. + */ +/* Order of the coprocessors registered first (apple_sleep_coproc_register_first()). */ +#define APPLE_SLEEP_FIRST_SMC 0 /* its warning and commit come before the GPU's stop */ +#define APPLE_SLEEP_FIRST_GPU 1 +#define APPLE_SLEEP_FIRST_ORDERS 2 + #if IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) /* * A coprocessor that can't be stopped until a sleep is certain, and can't be @@ -30,17 +41,13 @@ int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie /* * The same with no ASC to clear: @last_stop clears its own run bits and lowers its own power * domain, and runs before every other coprocessor's. The refusal checks leave that domain on. + * Those registered first run in the order of @order, an APPLE_SLEEP_FIRST_* value, not of + * their place in the table. */ -int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), - void *data); +int apple_sleep_coproc_register_first(struct device *dev, unsigned int order, + int (*last_stop)(void *data), void *data); void apple_sleep_coproc_unregister(struct device *dev); -/* - * During a suspend: a driver left something the DRAM-kept wake would stall on, so this - * sleep takes the deep path. False if no sleep driver can do that. - */ -bool apple_sleep_request_deep_once(void); - /* * For a registered coprocessor: the last of its stop that may still refuse the sleep, run * with interrupts off before any coprocessor's @last_stop. A negative return refuses, and the @@ -69,6 +76,61 @@ bool apple_sleep_coproc_stopped(struct device *dev); * registered refuses the sleep: stopped any earlier, a later refusal would strand it. */ bool apple_sleep_commit_stops_coprocs(void); +#else +static inline int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, + bool pcie_ans, int (*last_stop)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_coproc_register_first(struct device *dev, unsigned int order, + int (*last_stop)(void *data), void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_coproc_unregister(struct device *dev) +{ +} + +static inline int apple_sleep_coproc_set_last_check(struct device *dev, + int (*last_check)(void *data)) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_enclave_unregister(struct device *dev) +{ +} + +static inline void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ +} + +static inline bool apple_sleep_coproc_stopped(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_commit_stops_coprocs(void) +{ + return false; +} +#endif + +#if IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) +/* + * During a suspend: a driver left something the DRAM-kept wake would stall on, so this + * sleep takes the deep path. False if no sleep driver can do that. + */ +bool apple_sleep_request_deep_once(void); /* * The USB host sleeps in place for this sleep, as macOS's does: its always-on domain stays @@ -117,7 +179,6 @@ bool apple_sleep_debug_link_cut_peek(struct device *dev); bool apple_sleep_debug_link_cut(void); /* This sleep will end the debug link: for the SMC's driver, which warns the SMC after it. */ -bool apple_sleep_debug_link_up(void); /* * From the SMC's driver, in its noirq resume: the SMC woke the AP for the lid alone, and the @@ -171,58 +232,12 @@ void apple_sleep_restore_pins(void); */ int apple_sleep_engine_reset(struct device *dev); #else -static inline int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, - bool pcie_ans, int (*last_stop)(void *data), - void *data) -{ - return -EOPNOTSUPP; -} - -static inline int apple_sleep_coproc_register_first(struct device *dev, - int (*last_stop)(void *data), void *data) -{ - return -EOPNOTSUPP; -} - +/* Without the T8140 controller no sleep goes deep: false, as with no sleep driver. */ static inline bool apple_sleep_request_deep_once(void) { return false; } -static inline void apple_sleep_coproc_unregister(struct device *dev) -{ -} - -static inline int apple_sleep_coproc_set_last_check(struct device *dev, - int (*last_check)(void *data)) -{ - return -EOPNOTSUPP; -} - -static inline int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), - void *data) -{ - return -EOPNOTSUPP; -} - -static inline void apple_sleep_enclave_unregister(struct device *dev) -{ -} - -static inline void apple_sleep_coproc_set_ready(struct device *dev, bool ready) -{ -} - -static inline bool apple_sleep_coproc_stopped(struct device *dev) -{ - return false; -} - -static inline bool apple_sleep_commit_stops_coprocs(void) -{ - return false; -} - static inline void apple_sleep_usb_set_asleep(bool asleep) { } @@ -270,11 +285,6 @@ static inline bool apple_sleep_debug_link_cut(void) return false; } -static inline bool apple_sleep_debug_link_up(void) -{ - return false; -} - static inline void apple_sleep_lid_closed_wake(void) { } @@ -319,4 +329,22 @@ static inline int apple_sleep_engine_reset(struct device *dev) } #endif +/* + * Whether this firmware-driven suspend may end in a real power loss of the SoC, so that a + * driver may stop what only a power cycle brings back. A dry run, which the T6040 has, sets + * pm_suspend_via_firmware() to run every sleep step that can still be taken back and then + * refuses before the commit: there this is false, and such a stop has to be left out. The + * SMC's sleep commit is one of them: an SMC that took it stays in its sleep state until the + * SoC has lost its power and come back. + * Everywhere else it is pm_suspend_via_firmware(). + */ +#if IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) +bool apple_sleep_power_loss(void); +#else +static inline bool apple_sleep_power_loss(void) +{ + return pm_suspend_via_firmware(); +} +#endif + #endif diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 365437ce2cc02..f76d9d4a24cdb 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -422,6 +422,7 @@ extern void hibernation_set_ops(const struct platform_hibernation_ops *ops); extern int hibernate(void); extern bool system_entering_hibernation(void); extern bool hibernation_available(void); +void hibernation_set_platform_check(bool (*blocked)(void), const char *reason); asmlinkage int swsusp_save(void); extern struct pbe *restore_pblist; int pfn_is_nosave(unsigned long pfn); @@ -431,6 +432,23 @@ int hibernate_resume_nonboot_cpu_disable(void); int arch_hibernation_header_save(void *addr, unsigned int max_size); int arch_hibernation_header_restore(void *addr); +#ifdef CONFIG_ARCH_HIBERNATION_CLEAN_IMAGE +/** + * arch_hibernate_clean_image_page - Make a loaded image page visible to devices. + * @pfn: Valid page frame of the image. + * + * Provided by an architecture that sets ARCH_HIBERNATION_CLEAN_IMAGE. + * Called once for every valid page frame of the image after the whole image + * has been loaded, while the list of those frames is still intact, so that + * the architecture can clean the caches for devices that do not snoop them. + * + * Return: 1 when the page was cleaned, 0 when it was left alone because + * nothing was loaded into it through the linear map (the frame is restored + * from restore_pblist). + */ +int arch_hibernate_clean_image_page(unsigned long pfn); +#endif + #else /* CONFIG_HIBERNATION */ static inline void register_nosave_region(unsigned long b, unsigned long e) {} static inline int swsusp_page_is_forbidden(struct page *p) { return 0; } @@ -453,6 +471,16 @@ bool pm_hibernation_mode_is_suspend(void); static inline bool pm_hibernation_mode_is_suspend(void) { return false; } #endif +#ifdef CONFIG_HIBERNATION +bool pm_hibernate_is_restored(void); +bool pm_hibernate_is_loading(void); +int pm_hibernate_restored_cold(void); +#else +static inline bool pm_hibernate_is_restored(void) { return false; } +static inline bool pm_hibernate_is_loading(void) { return false; } +static inline int pm_hibernate_restored_cold(void) { return 0; } +#endif + int arch_resume_nosmt(void); #ifdef CONFIG_HIBERNATION_SNAPSHOT_DEV diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 890bf7b447f89..f771766758f0f 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include "power.h" @@ -81,6 +80,105 @@ static const struct platform_hibernation_ops *hibernation_ops; static atomic_t hibernate_atomic = ATOMIC_INIT(1); +/* Set when swsusp_arch_suspend() returned in a kernel restored from an image. */ +static bool hibernate_restored; + +/** + * pm_hibernate_is_restored - Whether this kernel was just restored from an image. + * + * Devices get PMSG_RESTORE in the kernel restored from a hibernation image, + * whose hardware went through a boot, and PMSG_RECOVER in the kernel that + * hibernated when hibernation is aborted before the image is created (a device + * failed to freeze, a wakeup event). True from the restore until the next + * hibernation starts (hibernation_snapshot()); meant for code that runs after + * the restore, such as the PM_POST_HIBERNATION notifiers. + */ +bool pm_hibernate_is_restored(void) +{ + return READ_ONCE(hibernate_restored); +} +EXPORT_SYMBOL_GPL(pm_hibernate_is_restored); + +/* + * What the kernel that loads an image leaves for the kernel restored from it. + * It lives in the no-save section, which the image neither saves nor + * overwrites, so the restored kernel reads what the loading kernel wrote, as + * long as both are the same build loaded at the same physical address (which + * the image's header requires). The record carries its own physical address + * and a magic: a record from a kernel loaded elsewhere, or none, reads as + * absent. + */ +struct hibernate_load_record { + u32 magic; + u32 other_boot; + u64 self_phys; +}; + +#define HIBERNATE_LOAD_MAGIC 0x484c4452 /* "HLDR" */ + +static struct hibernate_load_record hibernate_load_record __nosavedata; + +/* Set while hibernate() loads the image it has just written (test_resume). */ +static bool hibernate_loading_own_image; + +/* Whether the restore came from another boot, see pm_hibernate_restored_cold(). */ +static int hibernate_restored_cold = -ENODATA; + +/** + * pm_hibernate_restored_cold - Whether this kernel was restored by another boot. + * + * Meaningful once pm_hibernate_is_restored() is true, under the same lifetime. + * The kernel that loads an image tells the kernel restored from it whether it + * is the kernel that wrote the image (a test_resume load: the machine did not + * boot) or one that was started to resume it (the boot loader and the + * firmware have been through a boot). Whether a particular device lost its + * state across that boot is for its driver to find out. + * + * Return: 1 if the image was loaded by another boot, 0 if this kernel loaded + * its own image or was not restored, -ENODATA if the record of the loading + * kernel is missing. + */ +int pm_hibernate_restored_cold(void) +{ + if (!READ_ONCE(hibernate_restored)) + return 0; + return READ_ONCE(hibernate_restored_cold); +} +EXPORT_SYMBOL_GPL(pm_hibernate_restored_cold); + +/* Set while this kernel loads a hibernation image, see pm_hibernate_is_loading(). */ +static bool hibernate_loading; + +/** + * pm_hibernate_is_loading - Whether this kernel is loading a hibernation image. + * + * True from the PM_RESTORE_PREPARE notification, which starts the loading of an + * image (from the resume path or through the snapshot device), until the + * PM_POST_RESTORE notification that ends it when the image was not restored + * (or the notification chain refused it). The devices of the kernel that + * loads an image are quiesced for the jump into the image, not frozen for an + * image of their own; the flag lets drivers tell the two apart in their + * freeze callbacks. The kernel restored from an image reads false: the image + * holds the state of the kernel that created it. + */ +bool pm_hibernate_is_loading(void) +{ + return READ_ONCE(hibernate_loading); +} +EXPORT_SYMBOL_GPL(pm_hibernate_is_loading); + +/** + * hibernate_set_loading - Record that an image is being loaded or no longer is. + * @loading: New value of the flag read by pm_hibernate_is_loading(). + * + * Called from the PM notifier chain helpers around PM_RESTORE_PREPARE and + * PM_POST_RESTORE. + */ +void hibernate_set_loading(bool loading) +{ + WRITE_ONCE(hibernate_loading, loading); +} + #ifdef CONFIG_SUSPEND /** * pm_hibernation_mode_is_suspend - Check if hibernation has been set to suspend @@ -107,23 +205,36 @@ bool hibernation_in_progress(void) return !atomic_read(&hibernate_atomic); } -/* - * Apple silicon Macs run firmware on their coprocessors (storage, secure - * enclave, display, always-on processor, system controller, GPU and more). - * Restoring an image means a cold boot through the platform boot chain, which - * restarts that firmware, but most of the Linux drivers have no - * freeze/poweroff/restore path that rebuilds their mailbox, ring and DMA state - * from a cold start. The restored kernel would keep using stale state, so - * writing and restoring images stays disabled on these machines. +static bool (*hibernation_platform_blocked_fn)(void); +static const char *hibernation_platform_blocked_reason; + +/** + * hibernation_set_platform_check - Let platform code refuse hibernation. + * @blocked: Tells whether hibernation is refused on this platform at the time. + * @reason: Why it is refused, for the log. + * + * A platform whose drivers cannot yet restore their devices from an image + * registers a check that is true until the platform has what it needs. While + * it is true, writing and restoring images stays disabled + * (hibernation_available() is false) and @reason is logged at the end of the + * boot. Only one check can be registered. * * pm_test dry runs are allowed (see hibernate()): every pm_test level stops * before create_image() takes a snapshot, so no image is written or restored. * They exercise the freezer and the device freeze/thaw callbacks that a real * implementation needs. */ +void hibernation_set_platform_check(bool (*blocked)(void), const char *reason) +{ + if (WARN_ON_ONCE(hibernation_platform_blocked_fn)) + return; + hibernation_platform_blocked_reason = reason; + hibernation_platform_blocked_fn = blocked; +} + static bool hibernation_platform_blocked(void) { - return of_machine_is_compatible("apple,arm-platform"); + return hibernation_platform_blocked_fn && hibernation_platform_blocked_fn(); } static bool hibernation_generic_available(void) @@ -142,7 +253,7 @@ bool hibernation_available(void) static int __init hibernation_platform_report(void) { if (hibernation_platform_blocked()) - pr_info("disabled on this platform: coprocessor firmware state is not restored from an image\n"); + pr_info("disabled on this platform: %s\n", hibernation_platform_blocked_reason); return 0; } late_initcall(hibernation_platform_report); @@ -346,6 +457,25 @@ __weak int arch_resume_nosmt(void) return 0; } +/** + * hibernate_resume_event - Pick the event that resumes the devices frozen for an image. + * @error: Result of the image creation so far. + * + * Called once the image is created, restored or given up on. in_suspend is + * set only in the kernel that created the image, so a clear one means either + * that this kernel was restored from an image (devices get PMSG_RESTORE, + * their hardware went through a boot) or that the hibernation was aborted + * before the image existed (a device failed to freeze, a wakeup event, a + * pm_test level): the devices were only frozen, and the right event is + * PMSG_RECOVER, as after any other failed freeze. + */ +static pm_message_t hibernate_resume_event(int error) +{ + if (in_suspend) + return error ? PMSG_RECOVER : PMSG_THAW; + return pm_hibernate_is_restored() ? PMSG_RESTORE : PMSG_RECOVER; +} + /** * create_image - Create a hibernation image. * @platform_mode: Whether or not to use the platform driver. @@ -399,6 +529,13 @@ static int create_image(int platform_mode) if (!in_suspend) { events_check_enabled = false; clear_or_poison_free_pages(); + if (hibernate_load_record.magic == HIBERNATE_LOAD_MAGIC && + hibernate_load_record.self_phys == __pa_symbol(&hibernate_load_record)) + WRITE_ONCE(hibernate_restored_cold, !!hibernate_load_record.other_boot); + else + WRITE_ONCE(hibernate_restored_cold, -ENODATA); + hibernate_load_record.magic = 0; + WRITE_ONCE(hibernate_restored, true); } platform_leave(platform_mode); @@ -420,8 +557,7 @@ static int create_image(int platform_mode) Platform_finish: platform_finish(platform_mode); - dpm_resume_start(in_suspend ? - (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE); + dpm_resume_start(hibernate_resume_event(error)); return error; } @@ -438,6 +574,8 @@ int hibernation_snapshot(int platform_mode) int error; pm_suspend_clear_flags(); + WRITE_ONCE(hibernate_restored, false); + WRITE_ONCE(hibernate_restored_cold, -ENODATA); error = platform_begin(platform_mode); if (error) goto Close; @@ -485,7 +623,7 @@ int hibernation_snapshot(int platform_mode) if (error || !in_suspend) swsusp_free(); - msg = in_suspend ? (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE; + msg = hibernate_resume_event(error); dpm_resume(msg); if (error || !in_suspend) @@ -551,6 +689,9 @@ static int resume_target_kernel(bool platform_mode) save_processor_state(); error = restore_highmem(); if (!error) { + hibernate_load_record.other_boot = !hibernate_loading_own_image; + hibernate_load_record.self_phys = __pa_symbol(&hibernate_load_record); + hibernate_load_record.magic = HIBERNATE_LOAD_MAGIC; error = swsusp_arch_resume(); /* * The code below is only ever reached in case of a failure. @@ -558,6 +699,7 @@ static int resume_target_kernel(bool platform_mode) * swsusp_arch_suspend() was called. */ BUG_ON(!error); + hibernate_load_record.magic = 0; /* * This call to restore_highmem() reverts the changes made by * the previous one. @@ -774,6 +916,18 @@ static int load_image_and_restore(void) } error = swsusp_read(&flags); + if (!error) { + /* + * The image is in memory now. Clear its on-disk signature + * before the device restore, so that a restore which hangs + * (for example a co-processor that does not come back after + * the cold boot) cannot leave a valid image for the next boot + * to resume into the same hang. swsusp_close() releases the + * resume device, so do it while that is still open. + */ + if (swsusp_invalidate_sig()) + pr_warn("could not invalidate the image signature; a failed restore may repeat\n"); + } swsusp_close(); if (!error) error = hibernation_restore(flags & SF_PLATFORM_MODE); @@ -917,8 +1071,11 @@ int hibernate(void) if (snapshot_test) { pm_pr_dbg("Checking hibernation image\n"); error = swsusp_check(false); - if (!error) + if (!error) { + hibernate_loading_own_image = true; error = load_image_and_restore(); + hibernate_loading_own_image = false; + } } thaw_processes(); diff --git a/kernel/power/main.c b/kernel/power/main.c index 5429e9f19b655..ccced400c15c7 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -169,14 +169,25 @@ int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down) { int ret; + if (val_up == PM_RESTORE_PREPARE) + hibernate_set_loading(true); + ret = blocking_notifier_call_chain_robust(&pm_chain_head, val_up, val_down, NULL); + if (ret && val_up == PM_RESTORE_PREPARE) + hibernate_set_loading(false); + return notifier_to_errno(ret); } int pm_notifier_call_chain(unsigned long val) { - return blocking_notifier_call_chain(&pm_chain_head, val, NULL); + int ret = blocking_notifier_call_chain(&pm_chain_head, val, NULL); + + if (val == PM_POST_RESTORE) + hibernate_set_loading(false); + + return ret; } /* If set, devices may be suspended and resumed asynchronously. */ diff --git a/kernel/power/power.h b/kernel/power/power.h index 75b63843886ee..e43e77f9a999b 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -77,6 +77,7 @@ static inline void enable_restore_image_protection(void) {} #endif /* CONFIG_STRICT_KERNEL_RWX */ extern bool hibernation_in_progress(void); +void hibernate_set_loading(bool loading); #else /* !CONFIG_HIBERNATION */ @@ -84,6 +85,7 @@ static inline void hibernate_reserved_size_init(void) {} static inline void hibernate_image_size_init(void) {} static inline bool hibernation_in_progress(void) { return false; } +static inline void hibernate_set_loading(bool loading) {} #endif /* !CONFIG_HIBERNATION */ #define power_attr(_name) \ @@ -201,6 +203,7 @@ extern void swsusp_free(void); extern int swsusp_read(unsigned int *flags_p); extern int swsusp_write(unsigned int flags); void swsusp_close(void); +int swsusp_invalidate_sig(void); #ifdef CONFIG_SUSPEND extern int swsusp_unmark(void); #else diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index b209712cb2c3a..42d24f43991f8 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -36,6 +36,7 @@ #include #include #include +#include #include "power.h" @@ -2750,6 +2751,37 @@ static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca) return pbe->address; } +#ifdef CONFIG_ARCH_HIBERNATION_CLEAN_IMAGE +/* + * Most image pages are loaded straight into their original frames through + * the cacheable linear map (only conflicting pages go through restore_pblist). + * Where devices read memory without snooping the CPU caches, a later eviction + * of a dirty line would overwrite what a device wrote since: let the + * architecture clean every image page before the jump. + */ +static void swsusp_clean_restored_image(void) +{ + unsigned long pfn, n = 0, skipped = 0; + + memory_bm_position_reset(&orig_bm); + for (pfn = memory_bm_next_pfn(&orig_bm); pfn != BM_END_OF_MAP; + pfn = memory_bm_next_pfn(&orig_bm)) { + if (!pfn_valid(pfn)) + continue; + if (!arch_hibernate_clean_image_page(pfn)) { + skipped++; + continue; + } + if (!(++n & 0xffff)) + cond_resched(); + } + pr_debug("%lu image pages cleaned, %lu left to the restore_pblist copy\n", + n, skipped); +} +#else +static inline void swsusp_clean_restored_image(void) {} +#endif + /** * snapshot_write_next - Get the address to store the next image page. * @handle: Snapshot handle structure to guide the writing. @@ -2871,6 +2903,8 @@ int snapshot_write_finalize(struct snapshot_handle *handle) error = hibernate_restore_protect_page(handle->buffer); /* Do that only if we have loaded the image entirely */ if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages) { + /* while orig_bm still lists the image: recycling overwrites it */ + swsusp_clean_restored_image(); memory_bm_recycle(&orig_bm); free_highmem_data(); } diff --git a/kernel/power/swap.c b/kernel/power/swap.c index c78f1593600bb..8cc439448e053 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -483,11 +483,27 @@ static int flush_swap_writer(struct swap_map_handle *handle) static int swap_writer_finish(struct swap_map_handle *handle, unsigned int flags, int error) { + int flush; + if (!error) { pr_info("S"); error = mark_swapfiles(handle, flags); pr_cont("|\n"); flush_swap_writer(handle); + /* + * Make the image durable now: otherwise the device cache is + * flushed only by the device shutdown in the power-off that + * follows, and a platform whose power-off does not get that far + * loses the image. A failed flush leaves no durable image: drop + * the signature durably (unlike swsusp_unmark()) so that no + * later boot resumes from freed swap. + */ + flush = blkdev_issue_flush(file_bdev(hib_resume_bdev_file)); + if (!error && flush) { + if (swsusp_invalidate_sig()) + pr_err("image flush failed and the signature could not be dropped\n"); + error = flush; + } } if (error) @@ -1583,8 +1599,13 @@ int swsusp_check(bool exclusive) if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) { memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); swsusp_header_flags = swsusp_header->flags; - /* Reset swap signature now */ - error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, + /* + * Reset swap signature now. FUA: a REQ_SYNC write + * can sit in a volatile device cache, and a hard + * power-off of a hung restore then loses it and the + * next boot resumes the same image again. + */ + error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC | REQ_FUA, swsusp_resume_block, swsusp_header); } else { @@ -1625,6 +1646,39 @@ void swsusp_close(void) fput(hib_resume_bdev_file); } +/* + * Clear the on-disk image signature without freeing the swap pages. + * Called on the resume path right after the image is read into memory and + * before the device restore, so a restore that hangs (a co-processor that does + * not come back after a cold boot) cannot leave a valid image that the next + * boot resumes into the same hang. Unlike swsusp_unmark() this keeps the swap + * pages: the resuming kernel is about to jump into the restored image, and + * freeing swap here would desync the restored kernel's swap accounting. + */ +int swsusp_invalidate_sig(void) +{ + int error; + + error = hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block, swsusp_header); + if (error) + return error; + /* + * swsusp_check() normally reset the signature already; that write may + * still be in the device's volatile cache, so rewrite the header with + * FUA either way and make the reset durable before the restore. + */ + if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) + memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); + else if (memcmp(swsusp_header->sig, swsusp_header->orig_sig, 10)) + return -ENODEV; + error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC | REQ_FUA, + swsusp_resume_block, swsusp_header); + /* A device that ignores FUA still honours an explicit cache flush. */ + if (!error) + error = blkdev_issue_flush(file_bdev(hib_resume_bdev_file)); + return error; +} + /** * swsusp_unmark - Unmark swsusp signature in the resume device * diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 01e0e6cfbced8..3da0006a9c8ea 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -114,6 +114,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/suspend.c b/rust/helpers/suspend.c index a616907ecc0b5..9012a7f730569 100644 --- a/rust/helpers/suspend.c +++ b/rust/helpers/suspend.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include __rust_helper bool rust_helper_pm_suspend_via_firmware(void) @@ -11,3 +12,15 @@ __rust_helper bool rust_helper_pm_resume_via_firmware(void) { return pm_resume_via_firmware(); } + +/* An inline without CONFIG_HIBERNATION, so Rust reaches it here. */ +__rust_helper bool rust_helper_pm_hibernate_is_restored(void) +{ + return pm_hibernate_is_restored(); +} + +/* An inline on most configurations, so Rust reaches it here (linux/soc/apple/sleep.h). */ +__rust_helper bool rust_helper_apple_sleep_power_loss(void) +{ + return apple_sleep_power_loss(); +} diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index c34f187f4fb10..e4527ff598126 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -367,6 +367,15 @@ impl RtKit { to_result(unsafe { bindings::apple_rtkit_wake(self.rtk) }) } + /// Re-initialises a stopped session so a fresh HELLO/EPMAP boot can be + /// accepted again (hibernation restore of a coprocessor the boot loader + /// cold-restarted): detaches RX, frees inherited buffers, zeroes the + /// endpoint bitmap, reinitialises the completions and restarts the mailbox. + pub fn reinit(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_reinit(self.rtk) }) + } + /// Waits for the RTKit coprocessor to finish booting. pub fn boot(self: Pin<&mut Self>) -> Result { // SAFETY: `rtk` is valid per the type invariant. diff --git a/scripts/dtc/apple-dt-check.sh b/scripts/dtc/apple-dt-check.sh index 04b634e886427..820d8dc2daf41 100755 --- a/scripts/dtc/apple-dt-check.sh +++ b/scripts/dtc/apple-dt-check.sh @@ -2,7 +2,9 @@ # SPDX-License-Identifier: GPL-2.0-only # apple-dt-check.sh BASE HEAD WORKDIR: run CHECK_DTBS on every Apple device tree at BASE and at HEAD # and fail when HEAD has a warning BASE does not. Also runs dt_binding_check on every binding HEAD -# changes. Phandle values in the messages are masked, so renumbering is not a new warning. +# changes. Phandle values in the messages are masked, so renumbering is not a new warning, and so +# is the pattern list of a "does not match any of the regexes" message, which grows with every +# vendor prefix the bindings add. # Needs dtschema (dt-validate), clang/lld (LLVM=1), flex, bison. Prints a summary. set -u BASE=$1 HEAD=$2 W=$3 @@ -17,7 +19,7 @@ warnings() { # rev tag local rc=$? grep -E '\.dtb: |[Ww]arning|rror' "$W/$2.log" | grep -v '^ ' | sed -E "s#$t/##g; s#$o/##g" | - perl -pe 's{(\[\[[^:]*?\]\])}{(my $x = $1) =~ s/\d+/N/g; $x}ge; s{b\x27(?:\\.|[^\x27])*\x27}{b\x27N\x27}g' | + perl -pe 's{(\[\[[^:]*?\]\])}{(my $x = $1) =~ s/\d+/N/g; $x}ge; s{b\x27(?:\\.|[^\x27])*\x27}{b\x27N\x27}g; s{( not match any of the regexes): .*}{$1: ...}' | sort -u > "$W/$2.warn" echo "$2: make rc=$rc, $(wc -l < "$W/$2.warn") warning lines, $(echo $dtbs | wc -w) DTBs" return $rc diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index 07a47433ef4d9..37a3adae11431 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -23,7 +23,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -1256,6 +1258,96 @@ static int cs42l84_i2c_probe(struct i2c_client *i2c_client) return ret; } +/* + * System sleep. The interrupt is level low and its thread reads the codec over + * I2C, so it is off from the suspend until the codec is set up again. A real + * S2R cuts the codec's power, and a resume from disk went through a boot: the + * codec comes back with its reset defaults, or with what the kernel that + * loaded the image set up. Its registers are then written again from the + * cache, and a jack that was seen before is reported gone; one still plugged + * in raises the plug interrupt again. + */ +static int cs42l84_suspend(struct device *dev) +{ + struct cs42l84_private *cs42l84 = dev_get_drvdata(dev); + struct i2c_client *i2c_client = to_i2c_client(dev); + + if (i2c_client->irq) + disable_irq(i2c_client->irq); + scoped_guard(mutex, &cs42l84->irq_lock) + cs42l84_remote_stop(cs42l84); + cancel_delayed_work_sync(&cs42l84->remote_poll_work); + regcache_cache_only(cs42l84->regmap, true); + + return 0; +} + +static int cs42l84_wake(struct device *dev, bool lost) +{ + struct cs42l84_private *cs42l84 = dev_get_drvdata(dev); + struct i2c_client *i2c_client = to_i2c_client(dev); + int ret = 0; + + regcache_cache_only(cs42l84->regmap, false); + if (lost) { + if (cs42l84->reset_gpio) { + gpiod_set_value_cansleep(cs42l84->reset_gpio, 0); + usleep_range(1000, 2000); + gpiod_set_value_cansleep(cs42l84->reset_gpio, 1); + } + usleep_range(CS42L84_BOOT_TIME_US, CS42L84_BOOT_TIME_US * 2); + + scoped_guard(mutex, &cs42l84->irq_lock) { + regcache_mark_dirty(cs42l84->regmap); + /* LATCH_TO_VP first, as the plug detection set-up writes it. */ + ret = regcache_sync_region(cs42l84->regmap, CS42L84_MIC_DET_CTL4, + CS42L84_MIC_DET_CTL4); + if (!ret) + ret = regcache_sync(cs42l84->regmap); + if (ret) + dev_err(dev, "registers not restored after the sleep: %d\n", ret); + + if (cs42l84->tip_state != CS42L84_UNPLUG) { + cs42l84->tip_state = CS42L84_UNPLUG; + cs42l84->ring_state = CS42L84_UNPLUG; + cs42l84->hs_type = 0; + cs42l84_revert_hs(cs42l84); + if (cs42l84->jack) + snd_soc_jack_report(cs42l84->jack, 0, SND_JACK_HEADSET); + cs42l84_set_interrupt_masks(cs42l84, + CS42L84_RS_PLUG | CS42L84_RS_UNPLUG); + } + } + } else if (cs42l84->hs_type == SND_JACK_HEADSET) { + /* The headset is still there: poll its buttons again. */ + scoped_guard(mutex, &cs42l84->irq_lock) + cs42l84_remote_start(cs42l84); + } + if (i2c_client->irq) + enable_irq(i2c_client->irq); + + return ret; +} + +static int cs42l84_resume(struct device *dev) +{ + return cs42l84_wake(dev, pm_resume_via_firmware()); +} + +static int cs42l84_restore(struct device *dev) +{ + return cs42l84_wake(dev, true); +} + +static const struct dev_pm_ops cs42l84_pm_ops = { + .suspend = pm_sleep_ptr(cs42l84_suspend), + .resume = pm_sleep_ptr(cs42l84_resume), + .freeze = pm_sleep_ptr(cs42l84_suspend), + .thaw = pm_sleep_ptr(cs42l84_resume), + .poweroff = pm_sleep_ptr(cs42l84_suspend), + .restore = pm_sleep_ptr(cs42l84_restore), +}; + static void cs42l84_i2c_remove(struct i2c_client *i2c_client) { struct cs42l84_private *cs42l84 = i2c_get_clientdata(i2c_client); @@ -1284,6 +1376,7 @@ static struct i2c_driver cs42l84_i2c_driver = { .driver = { .name = "cs42l84", .of_match_table = cs42l84_of_match, + .pm = pm_sleep_ptr(&cs42l84_pm_ops), }, .id_table = cs42l84_id, .probe = cs42l84_i2c_probe, diff --git a/tools/asahi/apple-dart-fw/fixture.c b/tools/asahi/apple-dart-fw/fixture.c index 4ecb32bb7e512..0b0ed7e82127e 100644 --- a/tools/asahi/apple-dart-fw/fixture.c +++ b/tools/asahi/apple-dart-fw/fixture.c @@ -68,6 +68,8 @@ struct apple_dart { /* A J713 sleep's, never set here. */ void *sleep_record; bool cio_active; + phys_addr_t locked_phys[2][4]; + struct io_pgtable_cfg *locked_cfg[2]; }; struct apple_dart_stream_map {struct apple_dart*dart;unsigned long sidmap[1];}; struct apple_dart_atomic_stream_map {struct apple_dart*dart;unsigned long sidmap[1];}; diff --git a/tools/asahi/bcm4377-id-tests.py b/tools/asahi/bcm4377-id-tests.py index 167139cfffbc4..d1abc3413c749 100755 --- a/tools/asahi/bcm4377-id-tests.py +++ b/tools/asahi/bcm4377-id-tests.py @@ -37,40 +37,51 @@ def take(source, pattern): static int bcm4388_send_calibration(struct bcm4377_data *b) { (void)b;return 0; } struct pci_device_id { unsigned int vendor,device,subvendor,subdevice,class,class_mask;unsigned long driver_data; }; struct bcm4377_data { const struct bcm4377_hw *hw; }; -struct hci_dev { void (*reset)(struct hci_dev *); }; +struct hci_dev { void (*reset)(struct hci_dev *); bool no_suspend_notifier; }; +#define HCI_QUIRK_NO_SUSPEND_NOTIFIER 1 +#define hci_set_quirk(hdev, nr) ((hdev)->no_suspend_notifier = (nr) == HCI_QUIRK_NO_SUSPEND_NOTIFIER) static void bcm4377_hci_timeout(struct hci_dev *hdev) { (void)hdev; } static const char *board; -static bool of_machine_is_compatible(const char *compat) { assert(!strcmp(compat,"apple,j493"));return !strcmp(board,compat); } +static bool of_machine_is_compatible(const char *compat) { assert(!strcmp(compat,"apple,j493")||!strcmp(compat,"apple,j616s"));return !strcmp(board,compat); } ''' SUFFIX = r''' static bool probe_registration(struct bcm4377_data *bcm4377,const struct pci_device_id *id,struct hci_dev *hdev) { @SELECTION@ @REGISTRATION@ +@NOTIFIER@ return hdev->reset==bcm4377_hci_timeout; } int main(void) { unsigned cases=0; - const char *boards[]={"apple,j493","apple,j273","apple,j293","apple,j313","apple,j274","apple,j375","apple,j473","apple,j474","apple,j414s","apple,j416c","apple,j613","apple,j615","apple,j514s","apple,j700"}; + const char *boards[]={"apple,j493","apple,j273","apple,j293","apple,j313","apple,j274","apple,j375","apple,j473","apple,j474","apple,j414s","apple,j416c","apple,j613","apple,j615","apple,j514s","apple,j616s","apple,j700"}; assert(bcm4377_hw_variants[BCM4378].id==0x4378); assert(bcm4377_hw_variants[BCM4378].id!=4378); + assert(bcm4377_hw_variants[BCM4388].id==0x4388); + assert(bcm4377_hw_variants[BCM4388].id!=4388); for(unsigned chip=0;chip<4;chip++) { const struct pci_device_id *id=&bcm4377_devid_table[chip]; assert(id->vendor==PCI_VENDOR_ID_BROADCOM); assert(id->driver_data==chip); for(unsigned b=0;bdevice==BCM4378_DEVICE_ID && !strcmp(board,"apple,j493"); + bool expected=(id->device==BCM4378_DEVICE_ID && !strcmp(board,"apple,j493")) || + (id->device==BCM4388_DEVICE_ID && !strcmp(board,"apple,j616s")); assert(probe_registration(&data,id,&hdev)==expected); assert(data.hw==&bcm4377_hw_variants[id->driver_data]); assert(bcm4377_resume_recovery_supported(&data)==expected); if(expected)assert(hdev.reset==bcm4377_hci_timeout);else assert(!hdev.reset); + /* Only the pair whose suspend to RAM takes the firmware suspends and resumes HCI alone. */ + assert(hdev.no_suspend_notifier==(id->device==BCM4388_DEVICE_ID && !strcmp(board,"apple,j616s"))); cases++; } } board="apple,j493"; struct bcm4377_hw wrong=bcm4377_hw_variants[BCM4378];wrong.id=4378; struct bcm4377_data decimal={.hw=&wrong};assert(!bcm4377_resume_recovery_supported(&decimal));cases++; + board="apple,j616s"; + struct bcm4377_hw wrong88=bcm4377_hw_variants[BCM4388];wrong88.id=4388; + struct bcm4377_data decimal88={.hw=&wrong88};assert(!bcm4377_resume_recovery_supported(&decimal88));cases++; printf("%u table-backed identity and callback cases PASS\n",cases);return 0; } ''' @@ -95,8 +106,15 @@ def main(): 'registration': r'^\tif \(bcm4377_resume_recovery_supported\(bcm4377\)\)\n\t\thdev->reset = bcm4377_hci_timeout;', 'gate': r'^static bool bcm4377_resume_recovery_supported\([^;]*?\)\n\{.*?^\}', } + # What the current probe adds for a pair whose sleep takes the firmware; the old one has none. + notifier = take(current, r'^\tif \(bcm4377_resume_cold_sleep\(bcm4377\)\)\n\t\thci_set_quirk\(hdev, HCI_QUIRK_NO_SUSPEND_NOTIFIER\);') pieces = {name: take(current, pattern) for name, pattern in patterns.items()} old_pieces = {name: take(old, pattern) for name, pattern in patterns.items()} + # The admission function walks a table of chip and board pairs; the old one named its pair. + pieces['gate'] = '\n\n'.join([take(current, r'^static const struct bcm4377_resume_board \{.*?^\};'), + take(current, r'^static const struct bcm4377_resume_board \*bcm4377_resume_board\([^;]*?\)\n\{.*?^\}'), + pieces['gate'], + take(current, r'^static bool bcm4377_resume_cold_sleep\([^;]*?\)\n\{.*?^\}')]) # The old gate is compiled against the current tables. Later commits added # hardware fields and variant settings, so the struct and table may differ, # but the old gate must still see the same chip IDs, PCI table and probe code. @@ -120,17 +138,35 @@ def main(): assert 'BAR2_CONTEXT_ADDR_HI' in hw_defs, hw_defs source_prefix = PREFIX + defs + '\n' + hw_defs + '\n' + pci_defs + '\n' + any_id + '\n' + '\n'.join(pieces[n] for n in ('enum', 'hw_struct', 'variants', 'pci_macro_table')) suffix = SUFFIX.replace('@SELECTION@', pieces['selection']).replace('@REGISTRATION@', pieces['registration']) + # The published original has no such pair: its probe is compiled without the statement. + old_suffix = suffix.replace('@NOTIFIER@', '') + suffix = suffix.replace('@NOTIFIER@', notifier) gates = {'positive': pieces['gate'], 'published-original': old_pieces['gate']} assert pieces['gate'].count('0x4378') == 1 gates['decimal-id'] = pieces['gate'].replace('0x4378', '4378') gates['other-chip'] = pieces['gate'].replace('0x4378', '0x4387') - gates['board-bypass'] = pieces['gate'].replace('of_machine_is_compatible("apple,j493")', 'true') + # No row may match on any board, nor on the other row's. + assert pieces['gate'].count('of_machine_is_compatible(board->machine)') == 1 + gates['board-bypass'] = pieces['gate'].replace('of_machine_is_compatible(board->machine)', 'true') + assert pieces['gate'].count('"apple,j493"') == 1 and pieces['gate'].count('"apple,j616s"') == 1 + gates['j493-row-on-j616s'] = pieces['gate'].replace('"apple,j493"', '"apple,j616s"') + # The J616s row gets the same three, and the two rows must not trade boards. + assert pieces['gate'].count('0x4388') == 1 + gates['j616s-decimal-id'] = pieces['gate'].replace('0x4388', '4388') + gates['j616s-other-chip'] = pieces['gate'].replace('0x4388', '0x4387') + gates['j616s-row-on-j493'] = pieces['gate'].replace('"apple,j616s"', '"apple,j493"') + gates['crossed-boards'] = pieces['gate'].replace('"apple,j493"', '@').replace('"apple,j616s"', '"apple,j493"').replace('@', '"apple,j616s"') + # The row whose sleep takes the firmware is the J616s one, and only that one. + assert pieces['gate'].count(', .cold_sleep = true') == 1 + gates['cold-sleep-nowhere'] = pieces['gate'].replace(', .cold_sleep = true', '') + gates['cold-sleep-on-j493'] = pieces['gate'].replace(', .cold_sleep = true', '').replace('"apple,j493" }', '"apple,j493", .cold_sleep = true }') + assert gates['cold-sleep-on-j493'].count('.cold_sleep = true') == 1 results = {} def no_core(): resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) for name, gate in gates.items(): source = args.out / f'{name}.c' - source.write_text(source_prefix + gate + suffix) + source.write_text(source_prefix + gate + (old_suffix if name == 'published-original' else suffix)) binary = source.with_suffix('') subprocess.run([args.cc, '-std=gnu11', '-g', '-O1', '-fsanitize=address,undefined', '-fno-omit-frame-pointer', str(source), '-o', str(binary)], check=True) run = subprocess.run([str(binary)], capture_output=True, text=True, preexec_fn=no_core) diff --git a/tools/asahi/bcm4377-resume-tests.py b/tools/asahi/bcm4377-resume-tests.py index f8dfa488d5079..ee7ab21c04644 100755 --- a/tools/asahi/bcm4377-resume-tests.py +++ b/tools/asahi/bcm4377-resume-tests.py @@ -3,17 +3,19 @@ """Compile retained Bluetooth resume callbacks against controlled core/PCI services.""" import argparse, hashlib, json, os, re, resource, subprocess from pathlib import Path -p=argparse.ArgumentParser();p.add_argument('--out',required=True);p.add_argument('--old-ref');p.add_argument('--mutation',choices=['epoch','close','producer-fence','active-fence','board']);a=p.parse_args() +p=argparse.ArgumentParser();p.add_argument('--out',required=True);p.add_argument('--old-ref');p.add_argument('--mutation',choices=['epoch','close','producer-fence','active-fence','board','board-j616s','cold-gate','cold-windows','cold-running-fw','cold-rti','cold-unquiesce','cold-reset','cold-send','cold-queue','cold-unarmed','cold-rearm','cold-error','cold-sticky','cold-setup-retry','cold-defer','cold-open-queue','cold-owed-kept','cold-resume-fence','cold-queue-fence']);a=p.parse_args() r=Path(__file__).resolve().parents[2];o=Path(a.out);o.mkdir(parents=True,exist_ok=False);(o/'tmp').mkdir() def extract(s,n): - m=re.search(r'^(?:static )?(?:int|void|bool) '+n+r'\([^;]*?\)\n\{',s,re.M|re.S) + m=re.search(r'^(?:static )?(?:(?:int|void|bool) |const struct \w+ \*)'+n+r'\([^;]*?\)\n\{',s,re.M|re.S) if not m:raise ValueError(n) b=s.index('{',m.start())+1;depth=1 while depth:depth+=(s[b]=='{')-(s[b]=='}');b+=1 return s[m.start():b]+'\n' s=(r/'drivers/bluetooth/hci_bcm4377.c').read_text();core=(r/'net/bluetooth/hci_sync.c').read_text();core_hci=(r/'net/bluetooth/hci_core.c').read_text() -names=['bcm4377_setup_retry_allowed','bcm4377_queue_setup_retry','bcm4377_setup_retry_work','bcm4377_resume_recovery_supported','bcm4377_resume_recovery_allowed','bcm4377_resume_recovery_invalidate','bcm4377_resume_recovery_sync','bcm4377_resume_recovery_destroy','bcm4377_resume_recovery_work','bcm4377_hci_timeout','bcm4377_resume_recovery_stop','bcm4377_hci_open','bcm4377_hci_close','bcm4377_hci_setup_once','bcm4377_hci_setup','bcm4377_hci_post_init','bcm4377_hci_unregister_dev','bcm4377_sleep_finish','bcm4377_suspend','bcm4377_resume'] -bodies={n:extract(s,n) for n in names};shared=[] +names=['bcm4377_setup_retry_allowed','bcm4377_queue_setup_retry','bcm4377_setup_retry_work','bcm4377_resume_board','bcm4377_resume_recovery_supported','bcm4377_resume_recovery_allowed','bcm4377_resume_recovery_invalidate','bcm4377_resume_recovery_sync','bcm4377_resume_recovery_destroy','bcm4377_resume_recovery_work','bcm4377_hci_resume','bcm4377_hci_resume_work','bcm4377_queue_hci_resume','bcm4377_hci_timeout','bcm4377_resume_recovery_stop','bcm4377_hci_open','bcm4377_hci_close','bcm4377_hci_setup_once','bcm4377_hci_setup','bcm4377_hci_post_init','bcm4377_hci_send_frame','bcm4377_init_cfg','bcm4377_hci_unregister_dev','bcm4377_sleep_finish','bcm4377_suspend','bcm4377_resume_cold','bcm4377_resume'] +# The board table goes ahead of the functions that walk it. +boards=re.search(r'^static const struct bcm4377_resume_board \{.*?^\};',s,re.M|re.S).group()+'\n' +bodies={'@boards':boards,**{n:extract(s,n) for n in names}};shared=[] if a.old_ref: old=subprocess.check_output(['git','show',a.old_ref+':drivers/bluetooth/hci_bcm4377.c'],cwd=r,text=True) for n in ['bcm4377_hci_open','bcm4377_hci_close','bcm4377_hci_unregister_dev','bcm4377_suspend','bcm4377_resume']:bodies[n]=extract(old,n) @@ -23,10 +25,32 @@ def extract(s,n): 'close':('bcm4377_hci_close','bcm4377_resume_recovery_invalidate(bcm4377);','',18), 'producer-fence':('bcm4377_resume_recovery_stop','cancel_work_sync(&bcm4377->resume_work);','',21), 'active-fence':('bcm4377_resume_recovery_stop','flush_work(&bcm4377->hdev->cmd_sync_work);','',40), - 'board':('bcm4377_resume_recovery_supported','of_machine_is_compatible("apple,j493")','true',13), + 'board':('bcm4377_resume_board','of_machine_is_compatible(board->machine)','true',13), + # The BCM4388 row on the J493's board. + 'board-j616s':('@boards','"apple,j616s"','"apple,j493"',16), + # A resume that finds the function reset by the sleep and without firmware. + 'cold-gate':('bcm4377_resume','bcm4377_resume_recovery_supported(bcm4377) &&\n\t (bcm4377_resume_cold','(bcm4377_resume_cold',54), + 'cold-windows':('bcm4377_resume_cold','if (bcm4377_init_cfg(bcm4377))\n\t\treturn true;','',46), + 'cold-running-fw':('bcm4377_resume_cold','return bootstage != BCM4377_BOOTSTAGE_RUNNING ||\n\t rti_status != BCM4377_RTI_STATUS_RUNNING;','return true;',50), + 'cold-rti':('bcm4377_resume_cold','BCM4377_BOOTSTAGE_RUNNING ||','BCM4377_BOOTSTAGE_RUNNING &&',52), + 'cold-unquiesce':('bcm4377_resume','\t\tbcm4377->quiesced = false;\n\t\tbcm4377->needs_reset = true;','\t\tbcm4377->needs_reset = true;',46), + 'cold-reset':('bcm4377_resume','\t\tbcm4377->needs_reset = true;\n','',48), + 'cold-send':('bcm4377_hci_send_frame','READ_ONCE(bcm4377->resume_cold)','false',58), + 'cold-queue':('bcm4377_resume','queue_work(system_freezable_wq, &bcm4377->resume_work);','',46), + 'cold-unarmed':('bcm4377_resume','if (bcm4377->resume_armed) {','if (true) {',48), + 'cold-rearm':('bcm4377_resume','\t\t\tbcm4377->resume_armed = false;\n','',46), + 'cold-error':('bcm4377_resume','\t\treturn 0;\n\t}\n\n\tret = bcm4377_hci_resume(bcm4377);','\t}\n\n\tret = bcm4377_hci_resume(bcm4377);',47), + 'cold-sticky':('bcm4377_resume',' || READ_ONCE(bcm4377->resume_cold)','',61), + 'cold-setup-retry':('bcm4377_resume','\t\tbcm4377_queue_setup_retry(bcm4377);\n\t\treturn 0;','\t\treturn 0;',62), + # The core's resume waits for the open that boots the firmware, and is fenced like the recovery. + 'cold-defer':('bcm4377_resume','\t\tif (!READ_ONCE(bcm4377->hci_resume_owed))\n\t\t\tbcm4377_hci_resume(bcm4377);','\t\tbcm4377_hci_resume(bcm4377);',46), + 'cold-open-queue':('bcm4377_hci_post_init','\tbcm4377_queue_hci_resume(bcm4377);\n','',46), + 'cold-owed-kept':('bcm4377_hci_resume','\tif (!READ_ONCE(hdev->suspended))\n\t\tWRITE_ONCE','\tWRITE_ONCE',63), + 'cold-resume-fence':('bcm4377_resume_recovery_stop','\tcancel_work_sync(&bcm4377->hci_resume_work);\n','',64), + 'cold-queue-fence':('bcm4377_queue_hci_resume',' && !bcm4377->resume_blocked','',66), } if a.mutation: - n,old,new,case=mutations[a.mutation];assert old in bodies[n];bodies[n]=bodies[n].replace(old,new,1) + n,old,new,case=mutations[a.mutation];assert bodies[n].count(old)==1,(n,old);bodies[n]=bodies[n].replace(old,new,1) core_bodies={n:extract(core_hci,n) for n in ['hci_cmd_timeout','hci_cancel_cmd_sync','hci_suspend_dev','hci_resume_dev']} # The fixture models UP as a mask; the core uses a bit index. core_bodies={n:v.replace('test_bit(HCI_UP,', 'test_bit(HCI_UP_INDEX,') for n,v in core_bodies.items()} @@ -35,9 +59,13 @@ def extract(s,n): hw_enum=re.search(r'^enum bcm4377_chip \{.*?^\};',s,re.M|re.S).group() hw_table=re.search(r'^static const struct bcm4377_hw bcm4377_hw_variants\[\] = \{.*?^\};',s,re.M|re.S).group() fixture=re.sub(r'^struct bcm4377_hw \{.*?\};', 'typedef uint32_t u32;\n'+hw_struct,fixture,count=1,flags=re.M) -# The variant table names register offsets by macro; take their definitions from the driver. -hw_macros='\n'+''.join(m.group()+'\n' for name in sorted(set(re.findall(r'\b[A-Z][A-Z0-9_]+\b',hw_table))) for m in [re.search(r'^#define '+name+r'\b.*$',s,re.M)] if m) +# The variant table, the window setup and the cold-resume probe name registers by macro; take their definitions from the driver. +hw_macros='\n'+''.join(m.group()+'\n' for name in sorted(set(re.findall(r'\b[A-Z][A-Z0-9_]+\b',hw_table+extract(s,'bcm4377_init_cfg')+extract(s,'bcm4377_resume_cold')))) for m in [re.search(r'^#define '+name+r'\b.*$',s,re.M)] if m) assert 'BAR2_CONTEXT_ADDR_HI' in hw_macros,hw_macros +assert all(name in hw_macros for name in ('PCIECFG_BAR0_WINDOW1','PCIECFG_BAR2_WINDOW_DEFAULT','BAR2_BOOTSTAGE','BAR2_RTI_STATUS')),hw_macros +# The function reset is a fixture that ends the cold state on success; hold the driver to the same single place. +reset=extract(s,'bcm4377_pci_reset') +assert s.count('WRITE_ONCE(bcm4377->resume_cold, false)')==1 and re.search(r'^\tbcm4377->needs_reset = false;\n\tWRITE_ONCE\(bcm4377->resume_cold, false\);$',reset,re.M),'bcm4377_pci_reset' # bcm4377_suspend() undoes a failed wake arming through bcm4377_resume(), declared ahead of it. resume_decl=re.search(r'^static int bcm4377_resume\(struct device \*dev\);$',s,re.M).group()+'\n' fixture=fixture.replace('struct bcm4377_hw *hw;', 'const struct bcm4377_hw *hw;') @@ -56,10 +84,10 @@ def extract(s,n): (o/'compile.log').write_text(c.stdout+c.stderr);assert not c.returncode,c.stderr def no_core():resource.setrlimit(resource.RLIMIT_CORE,(0,0)) results=[] -for n in ([mutations[a.mutation][3]] if a.mutation else [0,1,2,28] if a.old_ref else range(46)): +for n in ([mutations[a.mutation][3]] if a.mutation else [0,1,2,28] if a.old_ref else range(67)): c=subprocess.run([str(o/'controls'),str(n)],capture_output=True,text=True,preexec_fn=no_core,env=dict(env,ASAN_OPTIONS='detect_leaks=1')) results.append({'case':n,'exit':c.returncode,'stdout':c.stdout,'stderr':c.stderr}) expected=-6 if a.old_ref or a.mutation else 0 -receipt={'source_sha256':hashlib.sha256(s.encode()).hexdigest(),'core_sha256':hashlib.sha256(core.encode()).hexdigest(),'core_hci_sha256':hashlib.sha256(core_hci.encode()).hexdigest(),'core_bodies':{n:hashlib.sha256(v.encode()).hexdigest() for n,v in core_bodies.items()},'actual_hw_struct_sha256':hashlib.sha256(hw_struct.encode()).hexdigest(),'actual_hw_table_sha256':hashlib.sha256(hw_table.encode()).hexdigest(),'table_callback_fixture':'Production variant table compiled unchanged; only calibration/PTB callback names map to controlled send boundaries.', 'extracted_bodies':{n:hashlib.sha256(v.encode()).hexdigest() for n,v in bodies.items()},'shared_new_prerequisites_for_old':shared,'old_ref':a.old_ref,'mutation':a.mutation,'core_fixture_adapter':'Only HCI_UP test_bit index is translated from fixture mask32 to index5; production conditions/order unchanged.', 'source_semantics':'Actual driver callbacks and synchronous close/open helper; core request serialization, PCI/firmware and HCI services are controlled fixtures, not whole-kernel or radio execution.','translation_sha256':hashlib.sha256(code.encode()).hexdigest(),'controls':results,'expected_exit':expected} +receipt={'source_sha256':hashlib.sha256(s.encode()).hexdigest(),'core_sha256':hashlib.sha256(core.encode()).hexdigest(),'core_hci_sha256':hashlib.sha256(core_hci.encode()).hexdigest(),'core_bodies':{n:hashlib.sha256(v.encode()).hexdigest() for n,v in core_bodies.items()},'actual_hw_struct_sha256':hashlib.sha256(hw_struct.encode()).hexdigest(),'actual_hw_table_sha256':hashlib.sha256(hw_table.encode()).hexdigest(),'table_callback_fixture':'Production variant table compiled unchanged; only calibration/PTB callback names map to controlled send boundaries.', 'extracted_bodies':{n:hashlib.sha256(v.encode()).hexdigest() for n,v in bodies.items()},'shared_new_prerequisites_for_old':shared,'old_ref':a.old_ref,'mutation':a.mutation,'core_fixture_adapter':'Only HCI_UP test_bit index is translated from fixture mask32 to index5; production conditions/order unchanged.', 'source_semantics':'Actual driver callbacks, config-window setup, cold-resume probe, send callback and synchronous close/open helper; core request serialization, PCI config space and BAR registers, the function reset with its firmware boot, and HCI services are controlled fixtures, not whole-kernel or radio execution.','reset_fixture':'bcm4377_pci_reset() is a controlled boundary; the driver source is held to clearing resume_cold only at the end of its success path.','translation_sha256':hashlib.sha256(code.encode()).hexdigest(),'controls':results,'expected_exit':expected} (o/'receipt.json').write_text(json.dumps(receipt,indent=2)+'\n');assert all(x['exit']==expected for x in results),results print(f'{len(results)} production controls passed') diff --git a/tools/asahi/bcm4377-resume/controls.c b/tools/asahi/bcm4377-resume/controls.c index ff558fd981ee8..2cd4318ece449 100644 --- a/tools/asahi/bcm4377-resume/controls.c +++ b/tools/asahi/bcm4377-resume/controls.c @@ -1,9 +1,116 @@ // SPDX-License-Identifier: GPL-2.0-only static bool timeout_during_resume; -static int hci_resume_sync(struct hci_dev *h) { if(timeout_during_resume)hci_cmd_timeout(&h->cmd_timer.work);return resume_error; } +static int resume_frames; +static char bars[2]; + +static int send(struct hci_dev *h, int type) +{ + unsigned char data[4] = {}; + struct sk_buff skb = { .pkt_type = type, .data = data, .len = sizeof(data) }; + + return bcm4377_hci_send_frame(h, &skb); +} + +/* + * The core resumes a controller it marked suspended, once: it sends its resume commands + * through the driver and carries on after one that fails. + */ +static int hci_resume_sync(struct hci_dev *h) +{ + int i; + + if (!h->suspended) + return 0; + h->suspended = false; + if (timeout_during_resume) + hci_cmd_timeout(&h->cmd_timer.work); + for (i = 0; i < resume_frames; i++) + send(h, HCI_COMMAND_PKT); + return resume_error; +} + +/* + * The sleep control register answers through BAR0. The firmware's status words are only + * there through the windows the driver sets, never through the ones a sleep reset. + */ +static int ioread32(void *addr) +{ + struct bcm4377_data *b = active; + char *vars = b->bar2 + b->hw->bar2_offset; + + if (addr == b->bar0 + BCM4377_BAR0_SLEEP_CONTROL) + return 0; + assert(!memcmp(cfg, cfg_probe, sizeof(cfg))); + if (addr == vars + BCM4377_BAR2_BOOTSTAGE) + return fw_bootstage; + assert(addr == vars + BCM4377_BAR2_RTI_STATUS); + return fw_rti_status; +} + +/* What probe leaves in the function's private windows. */ +static void probe_cfg(struct bcm4377_data *b) +{ + memset(cfg, 0, sizeof(cfg)); + assert(!bcm4377_init_cfg(b)); + memcpy(cfg_probe, cfg, sizeof(cfg)); + cfg_reads = 0; + cfg_writes = 0; +} + +/* A sleep that takes the function's power: a J616s reads window 1 as 0x19328000 after it. */ +static void power_loss(void) +{ + memset(cfg, 0, sizeof(cfg)); + cfg[BCM4377_PCIECFG_BAR0_WINDOW1 / 4] = 0x19328000; + fw_bootstage = 0; + fw_rti_status = 0; +} + +static int cold_cycle(struct device *dev) +{ + assert(!bcm4377_suspend(dev)); + power_loss(); + return bcm4377_resume(dev); +} + +static void j616s(struct bcm4377_hw *hw, struct bcm4377_data *b) +{ + board_j493 = false; + board_j616s = true; + *hw = bcm4377_hw_variants[BCM4388]; + probe_cfg(b); +} + +static int power_on(struct hci_dev *h) +{ + int ret; + + req_locked = true; + ret = hci_dev_open_sync(h); + req_locked = false; + return ret; +} + static int hci_dev_close_sync(struct hci_dev *h) {assert(req_locked);h->flags&=~HCI_UP;return bcm4377_hci_close(h);} static int hci_dev_open_sync(struct hci_dev *h) {assert(req_locked);int e=bcm4377_hci_open(h);if(!e)e=bcm4377_hci_post_init(h);if(!e)h->flags|=HCI_UP;return e;} static void deliver_work(struct bcm4377_data *b) {if(b->resume_work.queued){b->resume_work.queued=0;bcm4377_resume_recovery_work(&b->resume_work);}} + +/* The work that resumes the core after the open that booted the firmware. */ +static void deliver_hci_resume(struct bcm4377_data *b) +{ + if (b->hci_resume_work.queued) { + b->hci_resume_work.queued = 0; + bcm4377_hci_resume_work(&b->hci_resume_work); + } +} + +/* The whole recovery a cold resume queued: reset, open, then the core's resume. */ +static void cold_recover(struct bcm4377_data *b) +{ + deliver_work(b); + run_request(b->hdev); + deliver_hci_resume(b); +} static void resume_cycle(struct device *dev) {assert(!bcm4377_suspend(dev));assert(!bcm4377_resume(dev));} static void recover(struct bcm4377_data *b) {hci_cmd_timeout(&b->hdev->cmd_timer.work);deliver_work(b);run_request(b->hdev);} static void *active_callback(void *data) {run_request(data);return NULL;} @@ -13,6 +120,9 @@ int main(int argc,char **argv) { struct bcm4377_hw hw=bcm4377_hw_variants[BCM4378];struct pci_dev pci={}; struct bcm4377_data b={.hw=&hw,.pdev=&pci};struct hci_dev h={.data=&b,.flags=HCI_UP|HCI_RUNNING};b.hdev=&h;active=&b;pci.data=&b;struct device dev={.pci=&pci};pthread_mutex_init(&b.resume_lock,NULL);if(bcm4377_resume_recovery_supported(&b))h.reset=bcm4377_hci_timeout; bdaddr_t own={{2,3,4,5,6,7}},setup={{8,9,10,11,12,13}},pub={{14,15,16,17,18,19}};b.bdaddr=own;h.setup_addr=setup;h.public_addr=pub; + b.bar0 = bars; + b.bar2 = bars + 1; + probe_cfg(&b); switch(atoi(argv[1])) { case 0: resume_cycle(&dev);recover(&b);assert(reset_count==1&&cal_count==1&&ptb_count==1&&addr_count==1&&!bacmp(&h.bdaddr,&own)&&(h.flags&HCI_UP)&&!b.cal_needed);break; case 1: timeout_during_resume=true;resume_cycle(&dev);deliver_work(&b);run_request(&h);assert(reset_count==1);break; @@ -108,11 +218,348 @@ int main(int argc,char **argv) { recover(&b); assert(rebind_count == 1); break; + /* + * J616s: the BCM4388 is back from the sleep with its windows reset and no firmware. + * Nothing is sent to it: the resume queues the recovery, once, and leaves the core + * suspended. The core resumes after the open that booted the firmware. + */ + case 46: + j616s(&hw, &b); + resume_frames = 2; + assert(!cold_cycle(&dev)); + assert(!unquiesces && !b.quiesced); + assert(b.needs_reset && b.resume_cold && !memcmp(cfg, cfg_probe, sizeof(cfg))); + assert(!h.stat.err_tx && !h.stat.cmd_tx && !enqueued && !skbs_freed); + assert(h.suspended && b.hci_resume_owed && !b.hci_resume_work.queues); + assert(b.resume_work.queues == 1 && !reset_count && !request.fn); + hci_cmd_timeout(&h.cmd_timer.work); + assert(b.resume_work.queues == 1); + deliver_work(&b); + run_request(&h); + assert(reset_count == 1 && cal_count == 1 && ptb_count == 1 && addr_count == 1); + assert(!bacmp(&h.bdaddr, &own) && (h.flags & HCI_UP) && !b.cal_needed); + assert(!b.needs_reset && !b.resume_cold); + assert(h.suspended && b.hci_resume_owed && !enqueued); + assert(b.hci_resume_work.queues == 1); + deliver_hci_resume(&b); + assert(!h.suspended && !b.hci_resume_owed); + assert(enqueued == 2 && skbs_freed == 2 && !h.stat.err_tx); + assert(!send(&h, HCI_COMMAND_PKT) && enqueued == 3 && skbs_freed == 3); + recover(&b); + assert(reset_count == 1 && b.hci_resume_work.queues == 1); + break; + /* A cold resume succeeds whatever the core's resume returns; a warm one reports it. */ + case 47: + j616s(&hw, &b); + resume_error = -EIO; + assert(!cold_cycle(&dev)); + assert(b.resume_work.queues == 1); + cold_recover(&b); + assert(reset_count == 1 && (h.flags & HCI_UP) && !h.suspended); + assert(!bcm4377_suspend(&dev)); + assert(bcm4377_resume(&dev) == -EIO); + assert(unquiesces == 1 && !b.resume_cold && b.resume_work.queues == 1); + break; + /* + * A controller that was not powered gets no recovery and was never suspended: its + * next open resets the function, and the core has nothing to resume after it. + */ + case 48: + j616s(&hw, &b); + h.flags = 0; + resume_frames = 2; + assert(!cold_cycle(&dev)); + assert(b.needs_reset && b.resume_cold); + assert(!unquiesces && !b.resume_work.queues && !b.hci_resume_owed); + recover(&b); + assert(!reset_count && send(&h, HCI_COMMAND_PKT) == -ENODEV); + assert(!power_on(&h)); + assert(reset_count == 1 && cal_count == 1 && addr_count == 1 && (h.flags & HCI_UP)); + assert(!b.hci_resume_work.queues); + assert(!b.resume_cold && !send(&h, HCI_COMMAND_PKT) && enqueued == 1); + break; + /* A command timeout in the core's resume after the reset starts no second recovery. */ + case 49: + j616s(&hw, &b); + timeout_during_resume = true; + assert(!cold_cycle(&dev)); + assert(b.resume_work.queues == 1); + cold_recover(&b); + assert(reset_count == 1 && !b.resume_cold && !h.suspended); + assert(b.resume_work.queues == 1); + recover(&b); + assert(reset_count == 1); + break; + /* Windows the sleep reset over a firmware that still runs: put back, then as before. */ + case 50: + j616s(&hw, &b); + resume_frames = 2; + assert(!bcm4377_suspend(&dev)); + power_loss(); + fw_bootstage = 2; + fw_rti_status = 2; + assert(!bcm4377_resume(&dev)); + assert(unquiesces == 1 && !misdirected && !b.needs_reset && !b.resume_cold); + assert(!b.resume_work.queues && enqueued == 2 && !h.stat.err_tx); + assert(!h.suspended && !b.hci_resume_owed); + recover(&b); + assert(reset_count == 1 && !b.hci_resume_work.queues); + break; + /* Windows the sleep left alone: nothing is written, resume and recovery as before. */ + case 51: + j616s(&hw, &b); + resume_frames = 2; + resume_cycle(&dev); + assert(!cfg_writes && unquiesces == 1 && !b.needs_reset && !b.resume_cold); + assert(!b.resume_work.queues && enqueued == 2 && !h.stat.err_tx); + recover(&b); + assert(reset_count == 1 && cal_count == 1 && addr_count == 1); + break; + /* Either status word alone says no firmware runs, and each such sleep is recovered. */ + case 52: + j616s(&hw, &b); + assert(!bcm4377_suspend(&dev)); + power_loss(); + fw_bootstage = 2; + assert(!bcm4377_resume(&dev)); + assert(b.resume_cold && !unquiesces); + cold_recover(&b); + assert(!bcm4377_suspend(&dev)); + power_loss(); + fw_rti_status = 2; + assert(!bcm4377_resume(&dev)); + assert(b.resume_cold && !unquiesces); + cold_recover(&b); + assert(reset_count == 2 && cal_count == 2 && addr_count == 2); + assert(b.resume_work.queues == 2 && !b.resume_cold); + assert(b.hci_resume_work.queues == 2 && !h.suspended); + break; + /* The J493 pair takes the same path. */ + case 53: + resume_frames = 2; + assert(!cold_cycle(&dev)); + assert(!unquiesces && b.resume_cold && b.resume_work.queues == 1); + assert(!h.stat.err_tx && !enqueued); + cold_recover(&b); + assert(reset_count == 1 && cal_count == 1 && ptb_count == 1 && addr_count == 1); + assert((h.flags & HCI_UP) && !b.resume_cold); + assert(enqueued == 2 && !h.suspended); + break; + /* Any other chip and board pair is not looked at: its resume is the one before. */ + case 54: + hw = bcm4377_hw_variants[BCM4388]; + probe_cfg(&b); + assert(!cold_cycle(&dev)); + assert(!cfg_reads && !cfg_writes && unquiesces == 1); + assert(!b.needs_reset && !b.resume_cold && !send(&h, HCI_COMMAND_PKT)); + board_j493 = false; + board_j616s = true; + hw = bcm4377_hw_variants[BCM4378]; + probe_cfg(&b); + assert(!cold_cycle(&dev)); + assert(!cfg_reads && !cfg_writes && unquiesces == 2); + assert(!b.needs_reset && !b.resume_cold && !send(&h, HCI_COMMAND_PKT)); + recover(&b); + assert(!reset_count && !b.resume_work.queues && !b.hci_resume_work.queues); + break; + /* Windows that cannot be written back: no status word is read, the function is reset. */ + case 55: + j616s(&hw, &b); + assert(!bcm4377_suspend(&dev)); + power_loss(); + cfg_error = -EIO; + assert(!bcm4377_resume(&dev)); + cfg_error = 0; + assert(!unquiesces && b.needs_reset && b.resume_cold && b.resume_work.queues == 1); + cold_recover(&b); + assert(reset_count == 1 && !b.resume_cold && !h.suspended); + break; + /* The next suspend fences the recovery a cold resume queued, as work and as request. */ + case 56: + j616s(&hw, &b); + assert(!cold_cycle(&dev)); + assert(b.resume_work.queued); + assert(!cold_cycle(&dev)); + assert(!request.fn && b.resume_work.queues == 2); + deliver_work(&b); + assert(request.fn); + assert(!cold_cycle(&dev)); + assert(!request.fn && b.resume_work.queues == 3 && quiesces == 3 && !unquiesces); + assert(!reset_count && b.resume_cold && b.hci_resume_owed && h.suspended); + cold_recover(&b); + assert(reset_count == 1 && !b.resume_cold && (h.flags & HCI_UP)); + assert(b.hci_resume_work.queues == 1 && !h.suspended && !b.hci_resume_owed); + break; + /* + * A power-off before the queued recovery runs invalidates it; the next open resets, + * and the core resumes after that open. + */ + case 57: + j616s(&hw, &b); + resume_frames = 2; + assert(!cold_cycle(&dev)); + deliver_work(&b); + assert(request.fn); + h.flags &= ~HCI_UP; + bcm4377_hci_close(&h); + run_request(&h); + assert(!reset_count && b.resume_cold && b.needs_reset); + assert(b.hci_resume_owed && h.suspended && !b.hci_resume_work.queues); + assert(!power_on(&h)); + assert(reset_count == 1 && !b.resume_cold && (h.flags & HCI_UP)); + assert(b.hci_resume_work.queues == 1 && !enqueued); + deliver_hci_resume(&b); + assert(enqueued == 2 && !h.stat.err_tx && !h.suspended && !b.hci_resume_owed); + break; + /* The send callback: each frame reaches its ring, and none does while cold. */ + case 58: + j616s(&hw, &b); + assert(!send(&h, HCI_COMMAND_PKT) && enqueued_ring == &b.hci_h2d_ring); + assert(!send(&h, HCI_ACLDATA_PKT) && enqueued_ring == &b.acl_h2d_ring); + assert(!send(&h, HCI_SCODATA_PKT) && enqueued_ring == &b.sco_h2d_ring); + assert(enqueued == 3 && skbs_freed == 3 && h.stat.byte_tx == 12 && !h.stat.err_tx); + assert(h.stat.cmd_tx == 1 && h.stat.acl_tx == 1 && h.stat.sco_tx == 1); + assert(!cold_cycle(&dev)); + assert(send(&h, HCI_COMMAND_PKT) == -ENODEV); + assert(send(&h, HCI_ACLDATA_PKT) == -ENODEV); + assert(send(&h, HCI_SCODATA_PKT) == -ENODEV); + /* The core frees a frame the driver refused. */ + assert(enqueued == 3 && skbs_freed == 3 && h.stat.byte_tx == 12); + assert(h.stat.err_tx == 3 && h.stat.cmd_tx == 1 && h.stat.acl_tx == 1); + assert(h.stat.sco_tx == 1); + cold_recover(&b); + assert(!send(&h, HCI_COMMAND_PKT) && enqueued == 4 && skbs_freed == 4); + break; + /* + * A reset that fails leaves the function cold, the controller down and the core + * suspended; the open that succeeds ends all three. + */ + case 59: + j616s(&hw, &b); + resume_frames = 2; + assert(!cold_cycle(&dev)); + reset_error = -EIO; + cold_recover(&b); + assert(reset_count == 1 && !(h.flags & HCI_UP) && b.needs_reset && b.resume_cold); + assert(b.hci_resume_owed && h.suspended && !b.hci_resume_work.queues); + reset_error = 0; + assert(!power_on(&h)); + assert(reset_count == 2 && !b.resume_cold && (h.flags & HCI_UP)); + assert(b.hci_resume_work.queues == 1 && !enqueued && !h.stat.err_tx); + deliver_hci_resume(&b); + assert(enqueued == 2 && !h.stat.err_tx && !h.suspended && !b.hci_resume_owed); + assert(!send(&h, HCI_COMMAND_PKT)); + break; + /* Armed for wake: no QUIESCE went out, and the cold resume is the same. */ + case 60: + j616s(&hw, &b); + fixture_wakeup = true; + assert(!cold_cycle(&dev)); + assert(!quiesces && !unquiesces && !b.wake_armed); + assert(b.resume_cold && b.resume_work.queues == 1); + cold_recover(&b); + assert(reset_count == 1 && !b.resume_cold && !h.suspended); + break; + /* A sleep that leaves the windows alone does not end the cold state an earlier one left. */ + case 61: + j616s(&hw, &b); + assert(!cold_cycle(&dev)); + assert(b.resume_work.queued); + resume_cycle(&dev); + assert(b.resume_cold && b.needs_reset && !unquiesces); + assert(b.resume_work.queues == 2); + cold_recover(&b); + assert(reset_count == 1 && !b.resume_cold && (h.flags & HCI_UP)); + assert(!h.suspended && !b.hci_resume_owed); + break; + /* A setup retry the suspend held back is queued again by a cold resume too. */ + case 62: + j616s(&hw, &b); + h.flags = HCI_SETUP | HCI_AUTO_OFF; + cal_failures = 1; + assert(bcm4377_hci_setup(&h) == -ETIMEDOUT); + bcm4377_hci_close(&h); + assert(b.setup_retry_pending && b.setup_retry_work.work.queued); + assert(!cold_cycle(&dev)); + assert(b.resume_cold && !b.resume_work.queues && !b.hci_resume_owed); + assert(b.setup_retry_pending && b.setup_retry_work.work.queued); + break; + /* + * A controller powered off between the reset and the core's resume keeps its turn: + * the core resumes after the next open, which needs no reset. + */ + case 63: + j616s(&hw, &b); + resume_frames = 2; + assert(!cold_cycle(&dev)); + deliver_work(&b); + run_request(&h); + assert(reset_count == 1 && b.hci_resume_work.queued); + h.flags &= ~HCI_UP; + bcm4377_hci_close(&h); + deliver_hci_resume(&b); + assert(h.suspended && b.hci_resume_owed && !enqueued && !h.stat.err_tx); + assert(!power_on(&h)); + assert(reset_count == 1 && b.hci_resume_work.queues == 2); + deliver_hci_resume(&b); + assert(enqueued == 2 && !h.stat.err_tx && !h.suspended && !b.hci_resume_owed); + assert(!power_on(&h)); + assert(b.hci_resume_work.queues == 2); + break; + /* + * The next suspend fences a queued resume of the core, so it cannot run against the + * function the next sleep leaves cold: it follows that sleep's reset. + */ + case 64: + j616s(&hw, &b); + resume_frames = 2; + assert(!cold_cycle(&dev)); + deliver_work(&b); + run_request(&h); + assert(reset_count == 1 && b.hci_resume_work.queued); + assert(!cold_cycle(&dev)); + assert(!b.hci_resume_work.queued && b.hci_resume_owed && h.suspended); + assert(!enqueued && !h.stat.err_tx); + cold_recover(&b); + assert(reset_count == 2 && enqueued == 2 && !h.stat.err_tx); + assert(!h.suspended && !b.hci_resume_owed); + break; + /* After the reset, a sleep that keeps the firmware resumes the core itself. */ + case 65: + j616s(&hw, &b); + resume_frames = 2; + assert(!cold_cycle(&dev)); + deliver_work(&b); + run_request(&h); + assert(reset_count == 1 && b.hci_resume_work.queued); + resume_cycle(&dev); + assert(unquiesces == 1 && !b.hci_resume_work.queued); + assert(enqueued == 2 && !h.stat.err_tx && !h.suspended && !b.hci_resume_owed); + break; + /* An open that ends behind the suspend fence queues nothing; the resume does it. */ + case 66: + j616s(&hw, &b); + resume_frames = 2; + assert(!cold_cycle(&dev)); + deliver_work(&b); + h.flags &= ~HCI_UP; + bcm4377_hci_close(&h); + run_request(&h); + assert(!bcm4377_suspend(&dev)); + assert(!power_on(&h)); + assert(reset_count == 1 && !b.resume_cold && (h.flags & HCI_UP)); + assert(b.hci_resume_owed && !b.hci_resume_work.queues && !enqueued); + assert(!bcm4377_resume(&dev)); + assert(unquiesces == 1 && enqueued == 2 && !h.stat.err_tx); + assert(!h.suspended && !b.hci_resume_owed && !b.hci_resume_work.queues); + break; default:abort(); } if (atoi(argv[1]) == 6 || atoi(argv[1]) == 44 || atoi(argv[1]) == 45) assert(rebind_count == 1); else assert(!rebind_count); + /* Every queued resume of the core was delivered or fenced. */ + assert(!b.hci_resume_work.queued); assert(!request.fn);printf("case %s passed\n",argv[1]);return 0; } diff --git a/tools/asahi/bcm4377-resume/fixture.h b/tools/asahi/bcm4377-resume/fixture.h index 5eece3476e65f..b61f744965e60 100644 --- a/tools/asahi/bcm4377-resume/fixture.h +++ b/tools/asahi/bcm4377-resume/fixture.h @@ -34,9 +34,15 @@ #define BCM4377_BAR0_SLEEP_CONTROL_QUIESCE 1 #define BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE 0 #define system_freezable_wq 0 +#define BIT(n) (1U << (n)) +#define HCI_COMMAND_PKT 1 +#define HCI_ACLDATA_PKT 2 +#define HCI_SCODATA_PKT 3 #define container_of(p,t,m) ((t*)((char*)(p)-offsetof(t,m))) #define to_delayed_work(p) container_of(p,struct delayed_work,work) -struct work_struct { int queued; }; +struct work_struct { + int queued, queues; +}; struct delayed_work { struct work_struct work; }; typedef struct { unsigned char b[6]; } bdaddr_t; static const bdaddr_t zero={}; @@ -45,8 +51,31 @@ static void bacpy(bdaddr_t *a,const bdaddr_t *b) { *a=*b; } static int bacmp(const bdaddr_t *a,const bdaddr_t *b) { return memcmp(a,b,6); } struct bcm4377_data; struct firmware { const unsigned char *data; int size; }; -struct hci_dev { struct bcm4377_data *data; unsigned int flags; void *req_workqueue,*workqueue; struct work_struct power_on,cmd_sync_work,cmd_work; struct delayed_work cmd_timer,ncmd_timer; int req_lock,cmd_cnt,suspend_state,wake_reason,wake_addr_type; bdaddr_t wake_addr; int *req_skb; void (*reset)(struct hci_dev*); bdaddr_t setup_addr,public_addr,bdaddr; }; +struct hci_dev { + struct bcm4377_data *data; + unsigned int flags; + void *req_workqueue, *workqueue; + struct work_struct power_on, cmd_sync_work, cmd_work; + struct delayed_work cmd_timer, ncmd_timer; + int req_lock, cmd_cnt, suspend_state, wake_reason, wake_addr_type; + /* The core's own mark: set by its suspend, cleared by its resume when powered. */ + bool suspended; + bdaddr_t wake_addr; + int *req_skb; + void (*reset)(struct hci_dev *hdev); + bdaddr_t setup_addr, public_addr, bdaddr; + struct { + int err_tx, cmd_tx, acl_tx, sco_tx, byte_tx; + } stat; +}; struct ring { bool enabled; }; +/* bcm4377_hci_send_frame() names the transfer ring type; the fixture has one ring type. */ +#define bcm4377_transfer_ring ring +struct sk_buff { + int pkt_type; + unsigned char *data; + unsigned int len; +}; struct pci_dev { int irq, dev; void *data; @@ -59,12 +88,12 @@ struct bcm4377_data { bool setup_retry_used, setup_retry_pending, setup_retry_blocked; struct delayed_work setup_retry_work; bdaddr_t bdaddr; - char *bar0; + char *bar0, *bar2; pthread_mutex_t resume_lock; uint64_t resume_epoch, resume_pending; bool resume_armed, resume_blocked, resume_powered; - bool resume_guarded, resume_recovering; - struct work_struct resume_work; + bool resume_guarded, resume_recovering, resume_cold, hci_resume_owed; + struct work_struct resume_work, hci_resume_work; struct ring acl_d2h_ring,acl_h2d_ring,sco_d2h_ring,sco_h2d_ring,hci_d2h_ring,hci_h2d_ring,sco_event_ring,sco_ack_ring,hci_acl_event_ring,hci_acl_ack_ring; }; static int cal_count,ptb_count,reset_count,open_count,queued_count,addr_count,cal_failures,ptb_error,addr_error,reset_error,rings_error,close_error; @@ -91,10 +120,42 @@ static int bcm4377_check_bdaddr(struct bcm4377_data *b) { return 0; } static int bcm4377_destroy_transfer_ring(struct bcm4377_data *b,struct ring *r) { r->enabled=false;return close_error; } static int bcm4377_destroy_completion_ring(struct bcm4377_data *b,struct ring *r) { r->enabled=false;return close_error; } static int bcm4377_hci_set_bdaddr(struct hci_dev *h,const bdaddr_t *a) { addr_count++;if(addr_error)return addr_error;h->bdaddr=*a;return 0; } -static int bcm4377_pci_reset(struct bcm4377_data *b) { reset_count++;if(active_callback_pause){pthread_barrier_wait(&active_started);pthread_barrier_wait(&active_release);}if(reset_error)return reset_error;b->needs_reset=false;return 0; } + +/* What the function shows the driver: its private config windows and the firmware's status. */ +static uint32_t cfg[64], cfg_probe[64]; +static int cfg_reads, cfg_writes, cfg_error, fw_bootstage = 2, fw_rti_status = 2; + +/* + * Controlled boundary for the function reset, the window setup, the firmware boot and the + * RTI handshake. Like the driver's, a reset that succeeds ends the cold state. + */ +static int bcm4377_pci_reset(struct bcm4377_data *b) +{ + reset_count++; + if (active_callback_pause) { + pthread_barrier_wait(&active_started); + pthread_barrier_wait(&active_release); + } + if (reset_error) + return reset_error; + memcpy(cfg, cfg_probe, sizeof(cfg)); + fw_bootstage = 2; + fw_rti_status = 2; + b->needs_reset = false; + b->resume_cold = false; + return 0; +} + static int bcm4377_hci_open_rings(struct bcm4377_data *b) { open_count++;return rings_error; } static int mod_delayed_work(int wq,struct delayed_work *w,int delay) { w->work.queued=1;return 1; } -static int queue_work(void *wq,struct work_struct *w) { w->queued=1;queued_count++;return 1; } + +static int queue_work(void *wq, struct work_struct *w) +{ + w->queued = 1; + w->queues++; + queued_count++; + return 1; +} typedef uint64_t u64; #define spin_lock_irqsave(l,f) do { (f)=0;pthread_mutex_lock(l); } while(0) @@ -107,7 +168,16 @@ static pthread_t producer_thread; static void hci_cmd_timeout(struct work_struct *work); static struct bcm4377_data *active; static bool hdev_is_powered(struct hci_dev *h) {return (h->flags&HCI_UP) && !(h->flags&HCI_AUTO_OFF);} -static bool of_machine_is_compatible(const char *s) {assert(!strcmp(s,"apple,j493"));return board_j493;} +static bool board_j616s; + +static bool of_machine_is_compatible(const char *s) +{ + if (!strcmp(s, "apple,j616s")) + return board_j616s; + assert(!strcmp(s, "apple,j493")); + return board_j493; +} + static void *kmalloc(size_t n,int flags) {return allocation_error?NULL:malloc(n);} static void kfree(void *p) {free(p);} #define bt_dev_warn(...) ((void)0) @@ -122,7 +192,24 @@ static void cancel_delayed_work_sync(struct delayed_work *w) {if(w==&active->hde #define disable_delayed_work_sync cancel_delayed_work_sync static struct pci_dev *to_pci_dev(struct device *d) {return d->pci;} static void *pci_get_drvdata(struct pci_dev *p) {return p->data;} -static void iowrite32(int val,void *addr) {if(val==1){assert(!request.fn && !active->resume_work.queued && !active_running);quiesces++;}} + +/* An UNQUIESCE through windows a sleep reset does not reach the sleep control register. */ +static int unquiesces, misdirected; + +static void iowrite32(int val, void *addr) +{ + if (val == BCM4377_BAR0_SLEEP_CONTROL_QUIESCE) { + assert(!request.fn && !active->resume_work.queued && !active_running); + assert(!active->hci_resume_work.queued); + quiesces++; + return; + } + assert(val == BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE); + unquiesces++; + if (memcmp(cfg, cfg_probe, sizeof(cfg))) + misdirected++; +} + static bool fixture_wakeup; static bool device_may_wakeup(struct device *d) { return fixture_wakeup; } /* Not a J713 sleep: the radio sleep hooks do nothing, as on every other board. */ @@ -138,10 +225,59 @@ static void apple_radio_sleep_endpoint_resumed(struct device *d, int result) {} static void pci_save_state(struct pci_dev *p) {} static int enable_irq_wake(int irq) { return 0; } static void disable_irq_wake(int irq) {} -static int ioread32(void *addr) { return 0; } +static int ioread32(void *addr); + +static int pci_read_config_dword(struct pci_dev *p, int where, uint32_t *val) +{ + assert(p == active->pdev && !(where & 3) && where < 256); + cfg_reads++; + *val = cfg[where / 4]; + return 0; +} + +static int pci_write_config_dword(struct pci_dev *p, int where, uint32_t val) +{ + assert(p == active->pdev && !(where & 3) && where < 256); + if (cfg_error) + return cfg_error; + cfg_writes++; + cfg[where / 4] = val; + return 0; +} + +static int enqueued, skbs_freed; +static struct ring *enqueued_ring; + +static int hci_skb_pkt_type(struct sk_buff *skb) +{ + return skb->pkt_type; +} + +static int bcm4377_enqueue(struct bcm4377_data *b, struct ring *r, void *data, size_t len, + bool wait) +{ + assert(!wait); + enqueued++; + enqueued_ring = r; + return 0; +} + +static void kfree_skb(struct sk_buff *skb) +{ + skbs_freed++; +} + static int hci_suspend_dev(struct hci_dev *h); static int hci_resume_dev(struct hci_dev *h); -static void hci_unregister_dev(struct hci_dev *h) {assert(!request.fn&&!active->resume_work.queued&&!active_running);h->flags|=HCI_UNREGISTER;unregistered++;} + +static void hci_unregister_dev(struct hci_dev *h) +{ + assert(!request.fn && !active->resume_work.queued && !active_running); + assert(!active->hci_resume_work.queued); + h->flags |= HCI_UNREGISTER; + unregistered++; +} + static int hci_dev_close_sync(struct hci_dev *h); static int hci_dev_open_sync(struct hci_dev *h); int hci_reset_dev_sync(struct hci_dev *h); @@ -153,7 +289,16 @@ static int hci_skb_opcode(int *p) {return *p;} static bool mgmt_powering_down(struct hci_dev *h) {return h->flags&HCI_POWERING_DOWN;} static void hci_req_sync_lock(struct hci_dev *h) {assert(!req_locked);req_locked=true;} static void hci_req_sync_unlock(struct hci_dev *h) {assert(req_locked);req_locked=false;} -static int hci_suspend_sync(struct hci_dev *h) {return suspend_error;} + +/* The core marks the controller suspended unless its suspend fails. */ +static int hci_suspend_sync(struct hci_dev *h) +{ + if (suspend_error) + return suspend_error; + h->suspended = true; + return 0; +} + static int hci_resume_sync(struct hci_dev *h); static void hci_clear_wake_reason(struct hci_dev *h) {} static void mgmt_suspending(struct hci_dev *h,int state) {} diff --git a/tools/asahi/brcmf-suspend-tests.py b/tools/asahi/brcmf-suspend-tests.py index 41957747d20ca..a039c8ce27f27 100644 --- a/tools/asahi/brcmf-suspend-tests.py +++ b/tools/asahi/brcmf-suspend-tests.py @@ -180,6 +180,10 @@ def function(source, name): static void brcmf_pcie_sleep_finish(struct device *d) {(void)d;abort();} static void brcmf_bus_cancel_reset_work(struct brcmf_bus *b) {(void)b;abort();} static int brcmf_pcie_pm_leave_D3(struct device *dev); +static bool via_firmware,image_restored,queue_fails;static unsigned int queued; +static bool pm_resume_via_firmware(void) {return via_firmware;} +static bool pm_hibernate_is_restored(void) {return image_restored;} +static bool brcmf_pcie_restart_queue(struct pci_dev *p) {(void)p;if(queue_fails)return false;queued++;return true;} ''' CALLBACK_STUBS = r''' @@ -359,6 +363,16 @@ def function(source, name): probe_error=fail?-EIO:0;assert(brcmf_pcie_pm_leave_D3(&device)==probe_error); assert(removes==1 && probes==1 && ndev[0].stopped && msgbuf.tx_suspended);cases++; } + /* + * After a firmware suspend or the restore of a hibernation image the cold function's restart + * is queued, or done here if it can't be. + */ + for(int image=0;image<2;image++)for(int nomem=0;nomem<2;nomem++) { + reset();assert(!brcmf_pcie_pm_enter_D3(&device));retain=false; + via_firmware=!image;image_restored=image;queue_fails=nomem;queued=0;assert(!brcmf_pcie_pm_leave_D3(&device)); + assert(queued==!nomem && removes==nomem && probes==nomem); + via_firmware=image_restored=false;queue_fails=false;cases++; + } printf("%u regulatory, TX queue, worker retention and PCIe PM controls passed\n",cases); return 0; } diff --git a/tools/asahi/display26-tests.py b/tools/asahi/display26-tests.py index f0c993fe03624..e97d822de2f77 100644 --- a/tools/asahi/display26-tests.py +++ b/tools/asahi/display26-tests.py @@ -87,7 +87,7 @@ def main(): '-I'+str(tmp), '-I'+str(ROOT), str(tmp/'profile.c'), '-o', str(tmp/'profile')) run(str(tmp/'profile')) # Platform functions are mocked; the bridge's actual decisions and cleanup execute. - for name in ('device', 'export', 'mutex', 'of'): + for name in ('atomic', 'device', 'export', 'limits', 'mutex', 'notifier', 'of'): write(tmp, 'linux/'+name+'.h', '#include "mock.h"\n') write(tmp, 'linux/errno.h', '#define EPROBE_DEFER 517\n#define ENODEV 19\n#define EINVAL 22\n#define EBUSY 16\n#define EIO 5\n#define ETIMEDOUT 110\n#define EAGAIN 11\n') write(tmp, 'linux/kconfig.h', '#define IS_REACHABLE(x) 1\n#define IS_ENABLED(x) 1\n') @@ -95,9 +95,12 @@ def main(): write(tmp, 'linux/pm.h', '#include "mock.h"\n' 'struct dev_pm_ops { int (*suspend_late)(struct device *); int (*resume_early)(struct device *); };\n' '#define LATE_SYSTEM_SLEEP_PM_OPS(s, r) .suspend_late = (s), .resume_early = (r),\n') - write(tmp, 'linux/suspend.h', '#include \n' + write(tmp, 'linux/suspend.h', '#include "mock.h"\n' 'static inline bool pm_suspend_via_firmware(void) { return false; }\n' - 'static inline bool pm_resume_via_firmware(void) { return false; }\n') + 'static inline bool pm_resume_via_firmware(void) { return false; }\n' + '#define PM_POST_HIBERNATION 0x0002\n#define PM_POST_RESTORE 0x0006\n' + 'int register_pm_notifier(struct notifier_block *nb);\n' + 'int unregister_pm_notifier(struct notifier_block *nb);\n') # The bridge's sleep-commit stop: MMIO, RTKit and the sleep driver, reached only in a suspend. write(tmp, 'linux/io.h', '#include "mock.h"\n' 'static inline u32 readl_relaxed(const volatile void __iomem *a) { assert(!"no system sleep here"); return 0; }\n' @@ -114,18 +117,21 @@ def main(): '\t\t\t\t\t int (*f)(void *), void *x) { assert(!"no system sleep here"); return -EIO; }\n' 'static inline void apple_sleep_coproc_set_ready(struct device *d, bool r) { assert(!"no system sleep here"); }\n' 'static inline bool apple_sleep_coproc_stopped(struct device *d) { assert(!"no system sleep here"); return false; }\n' - 'static inline bool apple_sleep_commit_stops_coprocs(void) { assert(!"no system sleep here"); return false; }\n') + 'static inline bool apple_sleep_commit_stops_coprocs(void) { assert(!"no system sleep here"); return false; }\n' + 'static inline bool apple_sleep_power_loss(void) { assert(!"no system sleep here"); return false; }\n') write(tmp, 'linux/soc/apple/pmp-display.h', '#include \nbool apple_pmp_display_retained_board(void);\n') write(tmp, 'mock.h', '''#pragma once #include #include #include +#include #include #define dev_err(...) ((void)0) #define dev_warn(...) ((void)0) #define dev_dbg(...) ((void)0) #define dev_err_probe(d, e, ...) (e) -#define dev_info(...) ((void)0) +/* Takes its arguments, so that a value that is only logged counts as used. */ +static inline void dev_info(const void *dev, const char *fmt, ...) {} #include #define __iomem #include @@ -142,6 +148,14 @@ def main(): #define guard(x) struct mutex_guard JOIN(g,__LINE__) __attribute__((cleanup(leave))) = enter #define scoped_guard(x,m) for(bool once=true;once;) for(struct mutex_guard g __attribute__((cleanup(leave)))=enter(m);once;once=false) #define EXPORT_SYMBOL_GPL(x) +typedef struct { int counter; } atomic_t; +#define ATOMIC_INIT(i) { (i) } +static inline void atomic_inc(atomic_t *v){v->counter++;} +static inline int atomic_read(const atomic_t *v){return v->counter;} +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +struct notifier_block { int (*notifier_call)(struct notifier_block*,unsigned long,void*); int priority; }; +#define NOTIFY_DONE 0 #define DL_FLAG_AUTOREMOVE_CONSUMER 1 static struct device_node *phandle; static bool link_fail; static struct device_node *of_parse_phandle(struct device_node*n,const char*k,int index){return phandle;} @@ -168,6 +182,11 @@ def main(): int apple_pmp_sleep(const void*data){assert(!"no system sleep here");return -EIO;} int apple_pmp_sleep_prepare(const void*data,struct apple_pmp_commit_stop*stop){assert(!"no system sleep here");return -EIO;} int apple_pmp_wake(const void*data){assert(!"no system sleep here");return -EIO;} +int apple_pmp_restore(const void*data){assert(!"no hibernation here");return -EIO;} +/* The PM notifier chain: one block, registered while the PMP is. */ +static struct notifier_block *pm_notifier; +int register_pm_notifier(struct notifier_block*nb){assert(!pm_notifier);pm_notifier=nb;return 0;} +int unregister_pm_notifier(struct notifier_block*nb){assert(pm_notifier==nb);pm_notifier=NULL;return 0;} int main(void){ struct device_node n={1},wrong={2}; struct device supplier={&n,true,0},consumer={NULL,false,0}; int data; assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); @@ -190,7 +209,7 @@ def main(): assert(apple_pmp_set_device_power(0xf,5,1)==0);assert(calls==3); assert(apple_pmp_set_device_power(0xe,5,0)==0);assert(calls==4); apple_pmp_unregister(&wrong);assert(pmp_data==&data); - apple_pmp_unregister(&data);assert(!pmp_data && !pmp_device); + apple_pmp_unregister(&data);assert(!pmp_data && !pmp_device);assert(!pm_notifier); assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); puts("production PMP bridge supplier/ready/ack controls: PASS"); } diff --git a/tools/asahi/pmu-rtc-tests.py b/tools/asahi/pmu-rtc-tests.py index 062597ca80b56..02b271bc28e12 100644 --- a/tools/asahi/pmu-rtc-tests.py +++ b/tools/asahi/pmu-rtc-tests.py @@ -28,6 +28,8 @@ 'baku-limit': ('.max_delay = U32_MAX,', '.max_delay = APPLE_PMU_RTC_MAX_DELAY,'), 'set-no-round': ('(counter + BIT(APPLE_PMU_RTC_HZ_SHIFT - 1))', '(counter)'), 'set-negative': ('if (offset < 0 || offset', 'if (offset'), + 'restore-nothing': ('rtc_update_irq(rtc->clock, 1, RTC_IRQF | RTC_AF);', + 'if (false) rtc_update_irq(rtc->clock, 1, RTC_IRQF | RTC_AF);'), } PRELUDE = r''' @@ -57,6 +59,8 @@ #define kfree free #define guard(x) (void) #define dev_err(...) ((void)0) +#define dev_warn(...) ((void)0) +#define PM_SUSPEND_ON 0 #define PM_SUSPEND_TO_IDLE 1 #define RTC_IRQF 0x80 #define RTC_AF 0x20 @@ -333,7 +337,18 @@ firmware_suspend = true; assert(!apple_pmu_rtc_prepare(&dev)); firmware_suspend = false; + /* Hibernation is accepted. A platform power-off stops the alarm, and goes on if that fails. */ + pm_suspend_target_state = PM_SUSPEND_ON; + assert(!apple_pmu_rtc_prepare(&dev)); + assert(!apple_pmu_rtc_set_alarm(&dev, &alarm) && (map.bytes[map.ctrl] & 0x40)); + assert(!apple_pmu_rtc_poweroff(&dev)); + assert(!(map.bytes[map.ctrl] & 0x40) && (map.bytes[map.mask] & 1)); + map.ops = 0; map.fail = 1; + assert(!apple_pmu_rtc_poweroff(&dev)); pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + /* After a resume from disk the RTC core goes through its alarm again, without a PMU access of the driver's own. */ + reset(v); + assert(!apple_pmu_rtc_restore(&dev) && clock_model.events == 1 && map.ops == 0); } int main(void) { run_variant(BAKU); run_variant(ABBEY); run_variant(STOWE); run_variant(MAVERICK); diff --git a/tools/asahi/sep-suspend-errors-tests.py b/tools/asahi/sep-suspend-errors-tests.py index 2ae69ec2c32fc..23dc813080519 100644 --- a/tools/asahi/sep-suspend-errors-tests.py +++ b/tools/asahi/sep-suspend-errors-tests.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT """Compile actual SEP commit/error paths with host-only dependency stubs. -Extract current C stop/finish/prepare/noirq, provider poll-one and registration, -Rust wrappers/enter/leave/exit/kick, and the kernel polling macros verbatim. +Extract current C stop/finish/prepare/noirq, the PM notifier, provider poll-one and +registration, Rust wrappers/enter/leave/exit/kick, and the kernel polling macros verbatim. C fatal paths are observed wholly within C, never by unwinding a Rust frame. Virtual time/MMIO only test AP control decisions, not firmware timing or safety. No target access or kernel build occurs. @@ -88,6 +88,12 @@ def replace_once(source, before, after): 'msg = apple_mbox_read_msg(mbox);\n\t\tmbox->rx(mbox, msg, mbox->cookie);\n\t\tret = 2;'), 'post-send-wake-refusal': ('sep_pm_commit_stop', 'c->sent = true;', 'c->sent = true;\n\tif (pm_wakeup_pending()) return -EBUSY;'), + 'held-before-refusal': ('sep_pm_notify', 'if (event(SEP_PM_RESTORE) > 0) {', + 'sep_diag_sleep(true);\n\t\tif (event(SEP_PM_RESTORE) > 0) {'), + 'cold-as-warm': ('sep_pm_notify', 'if (cold) {', 'if (false) {'), + 'unrecorded-as-warm': ('sep_pm_notify', 'if (cold < 0) {', 'if (false) {'), + 'unrecorded-as-cold': ('sep_pm_notify', 'cold = pm_hibernate_restored_cold();', + 'cold = pm_hibernate_restored_cold() != 0;'), } C_PREFIX = r''' @@ -138,7 +144,12 @@ def replace_once(source, before, after): static struct apple_mbox_hw hw; static struct apple_mbox mbox; static int apple_sleep_coprocs_lock; -static void *apple_sleep; +#define APPLE_SLEEP_REG_ENCLAVE_ANY BIT(0) +#define APPLE_SLEEP_REG_BOUND BIT(8) +#define APPLE_SLEEP_REG_READY BIT(9) +/* The T8140 controller as its probe leaves the registry: bound, then ready. */ +static unsigned int apple_sleep_reg = APPLE_SLEEP_REG_BOUND | APPLE_SLEEP_REG_READY; +static unsigned int apple_sleep_dry; static struct { struct device *dev; int (*stop)(void *); void *data; } apple_sleep_enclave; @@ -154,6 +165,22 @@ def replace_once(source, before, after): static unsigned queued, consumed; static ktime_t now; static bool (*sep_pm_told)(void); +/* The PM notifier: its events go to the driver, and the diag state shows a sleep held. */ +struct notifier_block { int unused; }; +enum { NOTIFY_DONE, NOTIFY_OK, NOTIFY_BAD = 0x8002 }; +enum { PM_HIBERNATION_PREPARE = 1, PM_POST_HIBERNATION, PM_SUSPEND_PREPARE, PM_POST_SUSPEND, + PM_RESTORE_PREPARE, PM_POST_RESTORE }; +#define pr_err(...) ((void)0) +static int (*sep_pm_event)(int event); +static struct device *sep_pm_dev; +static int restored_cold, held, sleeps_ended, reprobes; +static bool restored; +static void sep_diag_sleep(bool entering) { + if (entering) held = 1; else if (held) { held = 0; ++sleeps_ended; } +} +static int pm_hibernate_restored_cold(void) { return restored_cold; } +static bool pm_hibernate_is_restored(void) { return restored; } +static void sep_pm_schedule_reprobe(struct device *d) { ++reprobes; } static bool yes(void) { return true; } static bool no(void) { return false; } static bool pm_suspend_via_firmware(void) { return firmware; } @@ -251,7 +278,7 @@ def replace_once(source, before, after): memset(&hw, 0, sizeof(hw)); mbox = (struct apple_mbox){ .regs = &hw, .hw = &hw, .rx = foreign_rx, .cookie = &device }; config_enabled = hyp = sep_pm_suspending = dart_present = dart_ready = map_ok = true; - firmware = tx_ready = true; apple_sleep = &device; + firmware = tx_ready = true; apple_sleep_reg = APPLE_SLEEP_REG_BOUND | APPLE_SLEEP_REG_READY; real_wake = pending_wake = late_wake = sleep_refuse_before_send = false; domain_error = property_error = address_error = read_error = gate_after = reads = 0; put_refs = maps = unmaps = lookups = prepares = restores = sends = closes = 0; @@ -260,13 +287,15 @@ def replace_once(source, before, after): resource_bytes = 4; close_gate = 1; early_gate = 0; read_error_after = 1; queued = consumed = 0; message_in_lock = matched_read = false; now = 0; fatal_armed = false; fatal_message[0] = 0; sep_pm_told = yes; + sep_pm_event = NULL; sep_pm_dev = &device; restored_cold = 0; + held = sleeps_ended = reprobes = 0; restored = false; } void parameter(int which, int value) { switch (which) { case 0: config_enabled = value; break; case 1: hyp = value; break; case 2: sep_pm_suspending = value; break; case 3: dart_present = value; break; case 4: map_ok = value; break; case 5: domain_error = value; break; - case 6: read_error = value; break; case 7: apple_sleep = value ? &device : NULL; break; + case 6: read_error = value; break; case 7: apple_sleep_reg = APPLE_SLEEP_REG_BOUND | (value ? APPLE_SLEEP_REG_READY : 0); break; case 8: gate_after = value; break; case 9: sep_pm_commit.registered = value; break; case 10: read_error_after = value; break; case 11: dart_ready = value; break; case 12: real_wake = value; break; case 13: hw.is_ascwrap_v8 = value; break; @@ -435,8 +464,50 @@ def replace_once(source, before, after): fixture(); sep_pm_told = NULL; assert(noirq() == 0 && !lookups); fixture(); sep_pm_told = no; assert(noirq() == 0 && !lookups); } +static int attached, last_event; +static int driver_event(int event) { + last_event = event; + // The driver reports a live session to drop for a cold boot, like the real one. + return event == SEP_PM_RESTORE ? attached : event == SEP_PM_HIBERNATE_COLD; +} +static void test_notifier(void) { + struct notifier_block nb; + // No driver registered: the chain goes on untouched. + fixture(); assert(sep_pm_notify(&nb, PM_RESTORE_PREPARE, NULL) == NOTIFY_DONE && !held); + // A kernel that attached the enclave refuses to load an image. The PM core sends no + // PM_POST_RESTORE to the notifier that refused, so nothing may be left held. + fixture(); sep_pm_event = driver_event; attached = 1; + assert(sep_pm_notify(&nb, PM_RESTORE_PREPARE, NULL) == NOTIFY_BAD); + assert(last_event == SEP_PM_RESTORE && !held && !sleeps_ended); + // One that did not goes on, and the end of the restore ends the hold. + attached = 0; + assert(sep_pm_notify(&nb, PM_RESTORE_PREPARE, NULL) == NOTIFY_OK && held); + assert(sep_pm_notify(&nb, PM_POST_RESTORE, NULL) == NOTIFY_OK); + assert(last_event == SEP_PM_HIBERNATE_WARM && !held && sleeps_ended == 1); + // The end of a hibernation: an aborted one or a hybrid sleep's wake was never restored. + fixture(); sep_pm_event = driver_event; restored_cold = 1; + assert(sep_pm_notify(&nb, PM_HIBERNATION_PREPARE, NULL) == NOTIFY_OK && held); + assert(sep_pm_notify(&nb, PM_POST_HIBERNATION, NULL) == NOTIFY_OK); + assert(last_event == SEP_PM_HIBERNATE_WARM && !held && sleeps_ended == 1 && !reprobes); + // Restored after a boot that loaded the image: the enclave restarted, the device is probed again. + restored = true; restored_cold = 1; + assert(sep_pm_notify(&nb, PM_POST_HIBERNATION, NULL) == NOTIFY_OK); + assert(last_event == SEP_PM_HIBERNATE_COLD && reprobes == 1); + // Restored by the kernel that wrote the image (test_resume): the enclave never restarted. + restored_cold = 0; reprobes = 0; + assert(sep_pm_notify(&nb, PM_POST_HIBERNATION, NULL) == NOTIFY_OK); + assert(last_event == SEP_PM_HIBERNATE_WARM && !reprobes); + // No record of the loading kernel: the session is dropped, no probe is queued. + restored_cold = -ENODATA; + assert(sep_pm_notify(&nb, PM_POST_HIBERNATION, NULL) == NOTIFY_OK); + assert(last_event == SEP_PM_HIBERNATE_COLD && !reprobes); + // A suspend holds and releases the same way. + assert(sep_pm_notify(&nb, PM_SUSPEND_PREPARE, NULL) == NOTIFY_OK && held); + assert(sep_pm_notify(&nb, PM_POST_SUSPEND, NULL) == NOTIFY_OK && !held && sleeps_ended == 2); +} int main(void) { test_prepare_and_cleanup(); test_unsent(); test_provider(); test_sent(); test_poll_contract(); + test_notifier(); puts("SEP C commit controls passed"); return 0; } #endif @@ -446,7 +517,7 @@ def replace_once(source, before, after): #![allow(dead_code, unused_imports, unused_variables)] use std::cell::{Cell, RefCell, RefMut}; use std::ffi::{c_int, c_void}; -use std::sync::atomic::{AtomicU32, Ordering, Ordering::Relaxed}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering, Ordering::Relaxed}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct Error(i32); impl Error { fn from_errno(n: i32) -> Self { assert!(n < 0); Self(n) } } type Result = std::result::Result; @@ -529,13 +600,15 @@ def replace_once(source, before, after): struct SepData { dev: kernel::device::Device, phase: AtomicU32, control: Lock, mbox: Lock>, gate: Cell, sleep_rx_held: Flag, endpoint: Cell, control_wq: Wq, + session_gone: AtomicBool, } impl SepData { fn new() -> Self { Self { dev: kernel::device::Device, phase: AtomicU32::new(PHASE_READY), control: Lock(RefCell::new(Control { busy: false, wake_ok: true })), mbox: Lock(RefCell::new(Some(Mbox))), gate: Cell::new(pm::SleepGate::Open), - sleep_rx_held: Flag(Cell::new(false)), endpoint: Cell::new(true), control_wq: Wq } + sleep_rx_held: Flag(Cell::new(false)), endpoint: Cell::new(true), control_wq: Wq, + session_gone: AtomicBool::new(false) } } fn endpoint_present(&self, _: u32) -> bool { self.endpoint.get() } fn sleep_gate(&self) -> pm::SleepGate { self.gate.get() } @@ -659,21 +732,27 @@ def sources(source, contracts, mutation): pm = (source / 'drivers/soc/apple/pm.rs').read_text() sep = (source / 'drivers/soc/apple/sep.rs').read_text() shim = (source / 'drivers/soc/apple/pm_shim.c').read_text() + events = re.findall(r'^#define SEP_PM_[A-Z_]+\s+\d+$', + (source / 'drivers/soc/apple/shim.h').read_text(), re.M) + assert len(events) == 6, events mailbox = (source / 'drivers/soc/apple/mailbox.c').read_text() sleep = (source / 'drivers/soc/apple/sleep.c').read_text() + registry = (source / 'drivers/soc/apple/sleep-registry.c').read_text() iopoll = (contracts / 'include/linux/iopoll.h').read_text() names = { 'shim': ['sep_pm_wait_gated', 'sep_pm_clear_dev_disable', 'sep_pm_dart_restore', 'sep_pm_commit_rx', 'sep_pm_commit_take_rx', 'sep_pm_commit_stop', - 'sep_pm_commit_prepare', 'sep_pm_commit_finish', 'sep_pm_suspend_noirq'], + 'sep_pm_commit_prepare', 'sep_pm_commit_finish', 'sep_pm_suspend_noirq', + 'sep_pm_notify'], 'mailbox': ['apple_mbox_poll_one_supported', 'apple_mbox_poll_one'], - 'sleep': ['apple_sleep_commit_stops_coprocs', 'apple_sleep_enclave_register', - 'apple_sleep_enclave_unregister'], + 'sleep': ['apple_sleep_power_loss'], + 'registry': ['apple_sleep_commit_stops_coprocs', 'apple_sleep_enclave_register', + 'apple_sleep_enclave_unregister'], 'pm': ['leave_for_commit', 'commit_finish', 'clear_dev_disable', 'dart_restore'], 'sep': ['sleep_enter', 'sleep_leave_for_commit', 'sleep_hold_rx', 'sleep_release_rx', 'sleep_exit', 'sleep_kick'], } - texts = dict(shim=shim, mailbox=mailbox, sleep=sleep, pm=pm, sep=sep) + texts = dict(shim=shim, mailbox=mailbox, sleep=sleep, registry=registry, pm=pm, sep=sep) bodies = {n: function(texts[k], n) for k, group in names.items() for n in group} if mutation: name, before, after = MUTATIONS[mutation] @@ -683,11 +762,12 @@ def sources(source, contracts, mutation): c = C_PREFIX + '\n' + '\n'.join(macro(iopoll, n) for n in macros) c += '\n' + '\n'.join(re.search(r'^#define ' + n + r'\s+.*$', shim, re.M).group() for n in ['SEP_PM_POLL_US', 'SEP_PM_TIMEOUT_US']) + c += '\n' + '\n'.join(events) start = shim.index('static struct sep_pm_commit {') c += '\n' + shim[start:shim.index('} sep_pm_commit;', start) + len('} sep_pm_commit;')] # Use the actual provider helper, with only removal/capability dependency state # injected by send for the post-send negative-poll scenarios. - c += '\n' + '\n'.join(bodies[n] for group in ['mailbox', 'sleep', 'shim'] for n in names[group]) + c += '\n' + '\n'.join(bodies[n] for group in ['mailbox', 'sleep', 'registry', 'shim'] for n in names[group]) c += '\n' + C_CONTROLS gate_start = pm.index('#[derive(Clone, Copy, Debug, PartialEq, Eq)]') gate_end = pm.index('/// Stages `request`', gate_start) diff --git a/tools/asahi/t6030-pmp-graph-tests.py b/tools/asahi/t6030-pmp-graph-tests.py index 4c85001acf1a8..2b44c999ab689 100644 --- a/tools/asahi/t6030-pmp-graph-tests.py +++ b/tools/asahi/t6030-pmp-graph-tests.py @@ -49,6 +49,8 @@ def main(): 'static int apple_pmp_report_wait_t6030_14(', 'int apple_pmp_report_wait_supplier_ready('] if 'static bool apple_pmp_report_running(' in source: names.insert(0, 'static bool apple_pmp_report_running(') + if 'static bool apple_pmp_report_hold_locked(' in source: + names.insert(names.index('static int apple_pmp_report_set_state('), 'static bool apple_pmp_report_hold_locked(') if 'static int apple_pmp_report_wait_t6030_bound(' in source: names.insert(names.index('static struct device_node *apple_pmp_report_t6030_entry('), 'static int apple_pmp_report_wait_t6030_bound(') # System sleep (apple,release-for-sleep), where the source has it. diff --git a/tools/asahi/tb-dp-repair-tests.py b/tools/asahi/tb-dp-repair-tests.py index 16932f99efa51..9e6651a046691 100644 --- a/tools/asahi/tb-dp-repair-tests.py +++ b/tools/asahi/tb-dp-repair-tests.py @@ -91,6 +91,8 @@ def function(text, name): static void list_del_init(struct list_head*l){} static bool tb_port_is_dpin(struct tb_port*p){return p->is_dpin;} static void tb_dp_tunnel_notify(struct tb_tunnel*t,bool active){} +/* Only a USB3 tunnel from the host router is reported; these cases retire DP tunnels. */ +static void tb_usb3_host_tunnel_changed(struct tb_tunnel*t,bool on){CHECK(t->type==TB_TUNNEL_DP && !on);} static void tb_tunnel_deactivate(struct tb_tunnel*t){} static void tb_detach_bandwidth_group(struct tb_port*p){} static void tb_switch_dealloc_dp_resource(struct tb_switch*s,struct tb_port*p){} diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c index 5db71a2f04bbd..0d898646821d0 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -93,6 +93,8 @@ struct dwc3_apple { /* Stopped for a deep sleep, and the state to bring back. */ bool sleep_stopped; enum dwc3_apple_state sleep_state; + /* Kept in place through a deep sleep, or lost in one; the fixtures do neither. */ + bool sleep_in_place, sleep_dead; /* Votes sleep DDR while the host is up; the fixtures leave it off. */ bool sleep_ddr; bool rpm_held, host_only; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 86a95e0f4178e..7669ecdc8e755 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1080,6 +1080,9 @@ def fn(n, name): role = 'static inline enum dwc3_apple_state dwc3_apple_state_for_role(' if role in files['state']: decls.append(block(files['state'], files['state'].index(role))) + kept = 'static inline enum dwc3_apple_state dwc3_apple_state_for_role_kept(' + if kept in files['state']: + decls.append(block(files['state'], files['state'].index(kept))) start = files['atc'].index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') end = files['atc'].index('\n};', start) + 3 decls.append(files['atc'][start:end]) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 4dcdc35ed409b..6de376bb895a5 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -481,6 +481,7 @@ def function(text, name): #define IS_ERR(p) ((intptr_t)(p)<0) #define ERR_PTR(e) ((void *)(intptr_t)(e)) #define dev_info(...) ((void)0) +#define dev_dbg(...) ((void)0) #define PTR_ERR(p) ((int)(intptr_t)(p)) #define IS_ERR_OR_NULL(p) (!(p)||IS_ERR(p)) #define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index f70e13245e35d..eae4f541e4dc1 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -415,7 +415,7 @@ def probe(root): """ helpers += '\n#define of_machine_is_compatible(name) (!strcmp((name), \"apple,t8140\"))\n#define BIT_ULL(n) (1ULL << (n))\n#define SN201202X_INT_DP_SID_UPDATE BIT_ULL(37)\n' helpers += block(core, 'static bool cd321x_uses_dp_sid(') + '\n' + block(core, 'static u64 cd321x_irq_mask(') + '\n' - unit += helpers + block(core, 'int tipd_init(') + r""" + unit += helpers + block(core, 'static int tipd_start_application(') + '\n' + block(core, 'int tipd_init(') + r""" static void probe_initial(struct cd321x *c) { initial(c);ordinary_port=true;phy_provider.safe_after_role=true;c->tps.irq=1;data.register_port=register_port;data.read_data_status=read_data_status;data.read_power_status=tps6598x_read_power_status;data.connect=connect; connect_error=irq_error=register_error=node_puts=mux_get_error=0;