diff --git a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml index 91ae5a3ce1bca..4c736d06122dd 100644 --- a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml +++ b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml @@ -53,6 +53,8 @@ properties: reg: maxItems: 1 + nonposted-mmio: true + "#address-cells": const: 1 diff --git a/Documentation/devicetree/bindings/chosen/aura,hibernate.yaml b/Documentation/devicetree/bindings/chosen/aura,hibernate.yaml new file mode 100644 index 0000000000000..8c8e9390b1e60 --- /dev/null +++ b/Documentation/devicetree/bindings/chosen/aura,hibernate.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/chosen/aura,hibernate.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Memory layout a boot object keeps the same on every boot + +maintainers: + - František Balázsy + +description: | + A hibernation image puts every saved page back into the frame it was taken + from, so the boot that resumes an image has to give the kernel the same RAM, + and keep the same ranges from it, as the boot that wrote the image. On Apple + silicon the boot firmware loads the boot object at an address that changes + from boot to boot, and what the boot object keeps alive after the hand-off + moves with it. A boot object that instead keeps everything from the start of + RAM up to a fixed end from the kernel, at the same addresses on every boot, + says so with aura,hibernate-layout. Without it the kernel neither hibernates + nor resumes. + + The boot firmware still places some firmware segments anew on every boot, + inside that range, and a kernel restored from an image holds the device tree + of the boot that wrote the image. The boot object therefore puts the device + tree of every boot at one address inside the range, given in + aura,hibernate-dt. The image does not contain that memory, so the restored + kernel reads there what the resuming boot placed. + + The properties are named after aura, the boot loader that defined them and + whose released versions publish them. + +select: + properties: + $nodename: + const: chosen + required: + - $nodename + +properties: + $nodename: + const: chosen + + aura,hibernate-layout: + $ref: /schemas/types.yaml#/definitions/uint64-array + description: + The range of RAM that is kept from the kernel and laid out the same on + every boot. + items: + - description: Physical address of the start of RAM. + - description: Physical address of the end of the range, exclusive. + + aura,hibernate-dt: + $ref: /schemas/types.yaml#/definitions/uint64 + description: + Physical address, inside the range of aura,hibernate-layout, of the + device tree the kernel is started with. The kernel reads at most 2 MiB + there. + +dependencies: + aura,hibernate-dt: [ 'aura,hibernate-layout' ] + +additionalProperties: true +... diff --git a/Documentation/devicetree/bindings/chosen/aurora,s2r-mailbox.yaml b/Documentation/devicetree/bindings/chosen/aurora,s2r-mailbox.yaml new file mode 100644 index 0000000000000..a48f9bbc5ae3e --- /dev/null +++ b/Documentation/devicetree/bindings/chosen/aurora,s2r-mailbox.yaml @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/chosen/aurora,s2r-mailbox.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Return path of a boot object that owns the CPU reset vectors + +maintainers: + - František Balázsy + +description: | + On Apple silicon the boot firmware locks every CPU's reset vector to the + boot object it started. A CPU that comes back from a power-off, and every + CPU after a suspend to RAM, therefore restarts in the boot object, not in the + kernel. A boot object that sends such a CPU back to the kernel says so with + this property: a secondary CPU returns through its spin-table release + address, and the boot CPU of a resume jumps, at EL2 with the MMU off, to the + address the kernel left in the mailbox. Its presence also tells the kernel + that its spin-table CPUs may leave the kernel and come back. + + The mailbox is four 64-bit words in the CPU's byte order: + 0: the magic value 0x3152325341525541 ("AURAS2R1"), written by the boot + object before the kernel starts + 1: the MPIDR of the only CPU that may take the resume address + 2: the resume address, set by the kernel before it suspends and cleared + by the boot object when a CPU takes it + 3: how many times the resume address was taken + +select: + properties: + $nodename: + const: chosen + required: + - $nodename + +properties: + $nodename: + const: chosen + + aurora,s2r-mailbox: + $ref: /schemas/types.yaml#/definitions/uint64 + description: + Physical address of the mailbox, inside the boot object's own memory, + which the kernel does not use. + +additionalProperties: true +... diff --git a/Documentation/devicetree/bindings/display/apple,t6041-display-power.yaml b/Documentation/devicetree/bindings/display/apple,t6041-display-power.yaml new file mode 100644 index 0000000000000..ab19f769ea9f4 --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,t6041-display-power.yaml @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,t6041-display-power.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T6040 display power words + +maintainers: + - František Balázsy + +description: | + The words a T6040 (M4 Pro) display coprocessor's driver writes to lower + its power domains after it stops the firmware for a suspend-to-RAM and to + raise them again on the wake. A display coprocessor names this node in its + apple,display-power property. The words belong to the power manager and + the display block, so no driver binds it. + + The internal display coprocessor has its power-state words in the SoC's + east power manager block, the external ones in the west block, each at its + own offsets; the display PMC word that counts the powered coprocessors is + shared by all three. The boot loader publishes one node for the internal + coprocessor and one that both external coprocessors name. + +properties: + compatible: + const: apple,t6041-display-power + + reg: + items: + - description: a register window, named by reg-names + - description: a register window, named by reg-names + + reg-names: + description: + pmc-dcp is the display PMC word that counts the powered coprocessors, + ps the power manager block that holds the coprocessors' power-state + words. The two may come in either order. + oneOf: + - items: + - const: pmc-dcp + - const: ps + - items: + - const: ps + - const: pmc-dcp + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + display-power@5048a9c14 { + compatible = "apple,t6041-display-power"; + reg = <0x5 0x048a9c14 0x0 0x8>, + <0x5 0x02280000 0x0 0x14000>; + reg-names = "pmc-dcp", "ps"; + }; + + display-power@502800000 { + compatible = "apple,t6041-display-power"; + reg = <0x5 0x02800000 0x0 0x14000>, + <0x5 0x048a9c14 0x0 0x8>; + reg-names = "ps", "pmc-dcp"; + }; + }; diff --git a/Documentation/devicetree/bindings/media/apple,prores.yaml b/Documentation/devicetree/bindings/media/apple,prores.yaml index 69be59afd4e70..827863013b202 100644 --- a/Documentation/devicetree/bindings/media/apple,prores.yaml +++ b/Documentation/devicetree/bindings/media/apple,prores.yaml @@ -248,8 +248,12 @@ allOf: maxItems: 1 iommus: maxItems: 1 + resets: + description: + The shared codec reset. It also clears the + translations of the engine's DART, so a board may + leave it out and have a hung engine refused instead. required: - - resets - reg-names - if: properties: diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 55d40f9a40cc7..dea7da061a73f 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -504,7 +504,7 @@ allOf: properties: reg: minItems: 8 - maxItems: 8 + maxItems: 9 interrupts: minItems: 6 maxItems: 6 diff --git a/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml index 3178de1a8b7e4..68ee0945b02d0 100644 --- a/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,t6040-atcphy.yaml @@ -172,6 +172,29 @@ properties: $ref: "#/$defs/apple,tunable" description: USB2 host tunables, applied to the bank0 region. + apple,tunable-cio-lane0: + $ref: "#/$defs/apple,tunable" + description: > + Lane 0 receiver defaults for CIO (USB4/Thunderbolt) mode, applied to the + core region when the port enters USB4 or Thunderbolt. CIO mode is offered + only when both lane tables are present. + + apple,tunable-cio-lane1: + $ref: "#/$defs/apple,tunable" + description: Lane 1 receiver defaults for CIO mode, applied to the core region. + + apple,tunable-cio-uc-regs: + $ref: "#/$defs/apple,tunable" + description: Optional PHY microcontroller defaults for CIO mode, core region. + + apple,tunable-cio-lane0-eq: + $ref: "#/$defs/apple,tunable" + description: Optional lane 0 receiver equalization for CIO mode, core region. + + apple,tunable-cio-lane1-eq: + $ref: "#/$defs/apple,tunable" + description: Optional lane 1 receiver equalization for CIO mode, core region. + required: - compatible - reg @@ -199,6 +222,11 @@ allOf: mode-switch: false port: false ports: false + apple,tunable-cio-lane0: false + apple,tunable-cio-lane1: false + apple,tunable-cio-uc-regs: false + apple,tunable-cio-lane0-eq: false + apple,tunable-cio-lane1-eq: false additionalProperties: false diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml index e2bec4d954de9..228151a45575d 100644 --- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml @@ -69,6 +69,14 @@ properties: $ref: /schemas/types.yaml#/definitions/uint32 description: The number of pins in this GPIO controller. + apple,resume-restore-all: + type: boolean + description: + After a sleep that powers the block down, every pin's configuration is + written back, not only the pins a driver has claimed, as the vendor + firmware does for this block. The pins in gpio-reserved-ranges are left + as the wake found them. + gpio-reserved-ranges: true interrupts: diff --git a/Documentation/devicetree/bindings/soc/apple/apple,sleep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,sleep.yaml new file mode 100644 index 0000000000000..d084121d3bfb1 --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,sleep.yaml @@ -0,0 +1,262 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,sleep.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC system sleep controls + +maintainers: + - Diljit Singh + - František Balázsy + +description: | + The registers a kernel running directly at EL2 on an Apple SoC writes to enter + and return from suspend-to-RAM and to power its secondary CPUs off. They belong + to several hardware blocks, some of which other drivers also use, so this node + overlaps those drivers' register ranges and the driver does not claim them. + Each SoC has its own compatible, as its power domains differ. Blocks that + several SoCs have are named alike. + + On the T6040 the sleep engine waits for the display pipes, so the node lists + the power states that have to be off before a sleep, as the ADT's + device-check-ps-b4-sleep table does. Each entry names a PMGR register block, + the offset of a display pipe's CPU power state and the offset of its last + sub-block; the states in between are 8 bytes apart. All of the T6040's + register blocks are needed to enter a suspend and to undo it. + +properties: + compatible: + enum: + - apple,t6040-sleep + - apple,t8132-sleep + - apple,t8140-sleep + + reg: + minItems: 11 + maxItems: 28 + + reg-names: + minItems: 11 + maxItems: 28 + + apple,wake-enable: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + The wake source IDs left enabled across suspend-to-RAM. Every other + source is masked. + + apple,apcie: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The PCIe host whose shared registers and tunables the storage controller's + link depends on. They are needed even with that host disabled. On the + T6040 the boot loader adds it, with those registers and tunables on the + host's node; without it the storage controller can't be stopped for a + sleep. + + apple,pinctrl: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The pin controller whose pins a deep sleep resets. Its pins are saved + before a sleep and put back after it, before the storage controller + starts. + + apple,pinctrl-skip-pins: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + Pins of that controller to leave alone across a sleep, neither saved nor + put back. + + apple,debug-port: + $ref: /schemas/types.yaml#/definitions/phandle + description: + The USB-C port controller that carries the debug link, stopped for a sleep. + + apple,display-coprocessors: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + The DCP and the external DCPs, which must be off before a commit, in that + order. + + apple,check-ps-b4-sleep: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + The display pipes whose power states have to be off before a sleep, one + entry per pipe. + items: + items: + - description: PMGR syscon + - description: offset of the pipe's CPU power state + - description: offset of its last sub-block's power state + + apple,check-ps-b4-sleep-names: + $ref: /schemas/types.yaml#/definitions/string-array + description: One name per apple,check-ps-b4-sleep entry. + +required: + - compatible + +allOf: + - if: + properties: + compatible: + const: apple,t6040-sleep + then: + properties: + reg: + items: + - description: CPU start/stop block (pmgr + 0x88000) + - description: AON watchdog + - description: control words of the three CPU complexes + - description: SoC power gate, wake and retention voters + - description: clock generator, the clock mux park words + - description: SPMI controller of the PMU (boot stage, notify, hibernate) + - description: second SPMI controller's notify word + - description: MasterMini block with the S2R timer + - description: SOC-East page with retention, fabric, DCS, SIO and other power words + - description: SOC page with the west axi2af, ATC, fabric and DCS power words + - description: AOP global block, S2R control and wake masks + reg-names: + items: + - const: cpu-start + - const: wdt + - const: cluster + - const: pwrgate + - const: clkgen + - const: spmi + - const: spmi-a0 + - const: mastermini + - const: soc-east + - const: soc-ps + - const: wake + apple,wake-enable: false + apple,pinctrl: false + apple,pinctrl-skip-pins: false + apple,debug-port: false + apple,display-coprocessors: false + required: + - reg + - reg-names + - apple,check-ps-b4-sleep + - apple,check-ps-b4-sleep-names + + - if: + properties: + compatible: + const: apple,t8132-sleep + then: + properties: + reg: + minItems: 25 + maxItems: 26 + reg-names: + minItems: 25 + items: + - const: ps + - const: pwrgate + - const: cpu-start + - const: cllt + - const: cluster + - const: commit + - const: wdt + - const: aon-misc + - const: wake + - const: amcc0 + - const: amcc1 + - const: pmc-dcp + - const: dvfm-e + - const: dvfm-p + - const: spmi + - const: pmc-voters + - const: dcs + - const: ps-aon + - const: spmi-a0 + - const: clk-mux + - const: mcc0 + - const: mcc1 + - const: ptd + - const: clk-perf + - const: cluster-dcs + - const: ps-prot + apple,check-ps-b4-sleep: false + apple,check-ps-b4-sleep-names: false + required: + - reg + - reg-names + - apple,display-coprocessors + + - if: + properties: + compatible: + const: apple,t8140-sleep + then: + properties: + reg: + minItems: 28 + reg-names: + items: + - const: ps + - const: pwrgate + - const: cpu-start + - const: cllt + - const: cluster + - const: commit + - const: wdt + - const: aon-misc + - const: wake + - const: amcc0 + - const: amcc1 + - const: amcc2 + - const: amcc3 + - const: pmc-dcp + - const: dvfm-e + - const: dvfm-p + - const: spmi + - const: pmc-voters + - const: dcs + - const: ps-aon + - const: spmi-a0 + - const: clk-mux + - const: mcc0 + - const: mcc1 + - const: mcc2 + - const: ptd + - const: clk-perf + - const: cluster-dcs + apple,check-ps-b4-sleep: false + apple,check-ps-b4-sleep-names: false + required: + - reg + - reg-names + - apple,wake-enable + - apple,display-coprocessors + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + sleep-controller@502888000 { + compatible = "apple,t6040-sleep"; + reg = <0x5 0x02888000 0x0 0x20>, + <0x5 0x0836c000 0x0 0x4000>, + <0x5 0x02820000 0x0 0x20>, + <0x5 0x0282c000 0x0 0x1000>, + <0x5 0x00000000 0x0 0x80000>, + <0x5 0x09014000 0x0 0x1000>, + <0x5 0x09188000 0x0 0x1000>, + <0x5 0x08900000 0x0 0x8000>, + <0x5 0x02280000 0x0 0xd000>, + <0x5 0x02800000 0x0 0x9000>, + <0x5 0x08004000 0x0 0x1000>; + reg-names = "cpu-start", "wdt", "cluster", "pwrgate", + "clkgen", "spmi", "spmi-a0", "mastermini", + "soc-east", "soc-ps", "wake"; + apple,check-ps-b4-sleep = <&pmgr_disp 0x0 0x58>; + apple,check-ps-b4-sleep-names = "disp"; + }; + }; diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml deleted file mode 100644 index bdbe2a5ccadaf..0000000000000 --- a/Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml +++ /dev/null @@ -1,158 +0,0 @@ -# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/soc/apple/apple,t8140-sleep.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Apple SoC system sleep controls - -maintainers: - - Diljit Singh - -description: | - The registers a kernel running directly at EL2 on an Apple SoC writes to enter - and return from suspend-to-RAM and to power its secondary CPUs off. They belong - to several hardware blocks, some of which other drivers also use, so this node - overlaps those drivers' register ranges and the driver does not claim them. - Each SoC has its own compatible, as its power domains differ. - -properties: - compatible: - enum: - - apple,t8132-sleep - - apple,t8140-sleep - - reg: - minItems: 25 - maxItems: 28 - - reg-names: - minItems: 25 - maxItems: 28 - - apple,wake-enable: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: - The wake source IDs left enabled across suspend-to-RAM. Every other - source is masked. - - apple,apcie: - $ref: /schemas/types.yaml#/definitions/phandle - description: - The PCIe host whose shared registers and tunables the storage controller's - link depends on. They are needed even with that host disabled. - - apple,pinctrl: - $ref: /schemas/types.yaml#/definitions/phandle - description: - The pin controller whose pins a deep sleep resets. Its pins are saved - before a sleep and put back after it, before the storage controller - starts. - - apple,pinctrl-skip-pins: - $ref: /schemas/types.yaml#/definitions/uint32-array - description: - Pins of that controller to leave alone across a sleep, neither saved nor - put back. - - apple,debug-port: - $ref: /schemas/types.yaml#/definitions/phandle - description: - The USB-C port controller that carries the debug link, stopped for a sleep. - - apple,display-coprocessors: - $ref: /schemas/types.yaml#/definitions/phandle-array - description: - The DCP and the external DCPs, which must be off before a commit, in that - order. - -required: - - compatible - - reg - - reg-names - - apple,display-coprocessors - -allOf: - - if: - properties: - compatible: - contains: - const: apple,t8140-sleep - then: - required: - - apple,wake-enable - properties: - reg: - minItems: 28 - maxItems: 28 - reg-names: - items: - - const: ps - - const: pwrgate - - const: cpu-start - - const: cllt - - const: cluster - - const: commit - - const: wdt - - const: aon-misc - - const: wake - - const: amcc0 - - const: amcc1 - - const: amcc2 - - const: amcc3 - - const: pmc-dcp - - const: dvfm-e - - const: dvfm-p - - const: spmi - - const: pmc-voters - - const: dcs - - const: ps-aon - - const: spmi-a0 - - const: clk-mux - - const: mcc0 - - const: mcc1 - - const: mcc2 - - const: ptd - - const: clk-perf - - const: cluster-dcs - - if: - properties: - compatible: - contains: - const: apple,t8132-sleep - then: - properties: - reg: - minItems: 25 - maxItems: 26 - reg-names: - minItems: 25 - items: - - const: ps - - const: pwrgate - - const: cpu-start - - const: cllt - - const: cluster - - const: commit - - const: wdt - - const: aon-misc - - const: wake - - const: amcc0 - - const: amcc1 - - const: pmc-dcp - - const: dvfm-e - - const: dvfm-p - - const: spmi - - const: pmc-voters - - const: dcs - - const: ps-aon - - const: spmi-a0 - - const: clk-mux - - const: mcc0 - - const: mcc1 - - const: ptd - - const: clk-perf - - const: cluster-dcs - - const: ps-prot - -additionalProperties: false diff --git a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml index 78dcee199f937..c730e4f46a020 100644 --- a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml @@ -50,6 +50,8 @@ properties: power-domains: maxItems: 1 + post-init-providers: true + ports: $ref: /schemas/graph.yaml#/properties/ports properties: diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index 562fb2f315e5c..b51948af8b09f 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -124,7 +124,8 @@ properties: description: The Type-C PHY whose lanes carry the router's CIO link. A host that power-cycles the router for a deep sleep takes the PHY through the - sleep with it. + sleep with it. On T6040 the router starts only while that PHY is in + CIO mode and tells it when it stops. phy-names: const: cio @@ -285,6 +286,8 @@ allOf: - nhi@1f00000 - interrupts - interrupt-names + - phys + - phy-names else: required: - iommu@a80000 diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 4775b267e1f77..ed0d32cde3614 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -207,6 +207,8 @@ patternProperties: description: Atmel Corporation "^auo,.*": description: AU Optronics Corporation + "^aura,.*": + description: aura boot loader for Apple silicon "^aurora,.*": description: Aurora Silicon "^auvidea,.*": diff --git a/MAINTAINERS b/MAINTAINERS index 3f919f3e327f3..70fe19771d80e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2093,7 +2093,9 @@ F: drivers/soc/apple/pmp.rs F: include/linux/mt7932.h F: include/linux/soc/apple/pmp-report.h F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml +F: Documentation/devicetree/bindings/chosen/aura,hibernate.yaml F: Documentation/devicetree/bindings/chosen/aurora,ctxr-exec-window.yaml +F: Documentation/devicetree/bindings/chosen/aurora,s2r-mailbox.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/devicetree/bindings/display/apple,t8140-display-power.yaml @@ -2110,10 +2112,10 @@ F: Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,sleep.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml -F: Documentation/devicetree/bindings/soc/apple/apple,t8140-sleep.yaml F: Documentation/devicetree/bindings/sound/apple,t6041-aop-audio.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml @@ -2136,6 +2138,7 @@ F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c F: drivers/soc/apple/sleep*.c +F: drivers/soc/apple/sleep-internal.h F: drivers/tty/serial/apple_dockchannel* F: include/linux/soc/apple/pci-apple-piodma.h F: include/linux/soc/apple/sleep.h diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 81f92a6fb50f7..224bba9710481 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2689,6 +2689,10 @@ config ARCH_HIBERNATION_HEADER def_bool y depends on HIBERNATION +config ARCH_HIBERNATION_CLEAN_IMAGE + def_bool y + depends on HIBERNATION + config ARCH_SUSPEND_POSSIBLE def_bool y diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index e81fb8b6b5d0d..4cf87185fe811 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -71,10 +71,24 @@ adj-height-mm = <216>; }; +/* + * The AOP firmware reports the lid angle to the SMC once a second on its own, + * and every report on this mailbox ends a suspend-to-idle period: keep its + * messages in the FIFO across system sleep. + */ +&aop_mbox { + apple,hold-rx-in-sleep; +}; + &aop_audio { apple,chassis-name = "J616"; }; +/* Suspend-to-RAM through the boot object has been tested on this board. */ +&t6040_sleep { + status = "okay"; +}; + &sound { compatible = "apple,j616-macaudio", "apple,macaudio"; model = "MacBook Pro J616"; @@ -388,3 +402,102 @@ &aes { status = "okay"; }; + +/* + * The ProRes encoder: the shared codec power state (below the always-on + * fabric domain), its DART and the PMP soc-device record SCODEC0 (27). + * No reset: the shared codec's reset also clears the DART's translations, + * so the driver refuses a hung engine instead. + */ +&ps_fab1_soc { + status = "okay"; +}; + +&ps_scodec { + status = "okay"; +}; + +&pmp_report { + pmp_report_scodec: report@1b { + compatible = "apple,t6041-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1b>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + }; +}; + +&apr0_dart { + status = "okay"; +}; + +&apr0 { + power-domains = <&ps_scodec>, <&pmp_report_scodec>; + power-domain-names = "core", "perf"; + /delete-property/ resets; + status = "okay"; +}; + +/* + * The JPEG engines share the PMP soc-device record JPEG0 (26): hold it as + * their performance request while an engine runs. + */ +&pmp_report { + pmp_report_jpg: report@1a { + compatible = "apple,t6041-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1a>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + }; +}; + +&jpeg0 { + power-domains = <&ps_jpg>, <&pmp_report_jpg>; + power-domain-names = "core", "perf"; +}; + +&jpeg1 { + power-domains = <&ps_jpg>, <&pmp_report_jpg>; + power-domain-names = "core", "perf"; +}; + +/* + * DP/HDMI audio: the clients behind dcpext0/dcpext1, their power states and + * the SIO the boot loader publishes. Neither side may hold the other at + * probe: audio needs the SIO only when a stream opens, and the external + * displays must come up whether the audio clients bind or not. Only + * properties are added to the display processors: their status belongs to + * the display gate. + */ +&ps_dpa_p { + status = "okay"; +}; + +&ps_dpa1 { + status = "okay"; +}; + +&ps_dpa2 { + status = "okay"; +}; + +&dpaudio1 { + post-init-providers = <&sio>; + status = "okay"; +}; + +&dpaudio2 { + post-init-providers = <&sio>; + status = "okay"; +}; + +&dcpext0 { + post-init-providers = <&dpaudio1>; +}; + +&dcpext1 { + post-init-providers = <&dpaudio2>; +}; diff --git a/arch/arm64/boot/dts/apple/t6040-usb4.dtsi b/arch/arm64/boot/dts/apple/t6040-usb4.dtsi index cd3b2c21aca7a..cf75493f031b3 100644 --- a/arch/arm64/boot/dts/apple/t6040-usb4.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-usb4.dtsi @@ -36,6 +36,9 @@ pinctrl-names = "lstx"; pinctrl-0 = <&usb4_0_lstx_pins>; lstx-gpios = <&pinctrl_aop 49 GPIO_ACTIVE_HIGH>; + /* The router starts only over its port's Type-C PHY in CIO mode. */ + phys = <&atcphy0 PHY_TYPE_USB2>; + phy-names = "cio"; power-domains = <&ps_atc0_cio>, <&ps_atc0_cio_pcie>, <&ps_atc0_cio_usb>, <&ps_atc0_pcie>; thunderbolt-switch; @@ -133,6 +136,9 @@ pinctrl-names = "lstx"; pinctrl-0 = <&usb4_1_lstx_pins>; lstx-gpios = <&pinctrl_aop 51 GPIO_ACTIVE_HIGH>; + /* The router starts only over its port's Type-C PHY in CIO mode. */ + phys = <&atcphy1 PHY_TYPE_USB2>; + phy-names = "cio"; power-domains = <&ps_atc1_cio>, <&ps_atc1_cio_pcie>, <&ps_atc1_cio_usb>, <&ps_atc1_pcie>; thunderbolt-switch; @@ -230,6 +236,9 @@ pinctrl-names = "lstx"; pinctrl-0 = <&usb4_2_lstx_pins>; lstx-gpios = <&pinctrl_aop 53 GPIO_ACTIVE_HIGH>; + /* The router starts only over its port's Type-C PHY in CIO mode. */ + phys = <&atcphy2 PHY_TYPE_USB2>; + phy-names = "cio"; power-domains = <&ps_atc2_cio>, <&ps_atc2_cio_pcie>, <&ps_atc2_cio_usb>, <&ps_atc2_pcie>; thunderbolt-switch; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 2cc327c7a2f8f..37ce82fc38171 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -9,6 +9,10 @@ /* Blocks and properties the T6041 tree does not describe yet. */ / { + aliases { + sio = &sio; + }; + opp_p0: opp-table-p0 { compatible = "operating-points-v2"; opp-shared; @@ -241,9 +245,70 @@ pmgr_disp: power-management@502290000 { compatible = "apple,t6041-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; reg = <0x5 0x02290000 0x0 0x4000>; + /* DISP_CPU's power word takes non-posted accesses (nGnRnE). */ + nonposted-mmio; #address-cells = <1>; #size-cells = <1>; }; + + t6040_sleep: sleep-controller@502888000 { + compatible = "apple,t6040-sleep"; + /* + * The SoC blocks the commit and wake + * write, all shared read/write with their owning drivers and + * touched only once no blocker remains: + * cpu-start pmgr reg0 + 0x88000 (misc-cores): CPU start/stop + * wdt the AON watchdog, the sleep's WD2 safety net + * cluster 0x502820000: the three CPU complexes E/P0/P1 + * pwrgate 0x50282c000: wake + CPM retention voters + * clkgen 0x500000000: the clk-mux park words + * spmi 0x509014000: the PMU boot-stage + NOTIFY + go-hib + * spmi-a0 0x509188000: the second SPMI NOTIFY + * mastermini 0x508900000: the S2R timer at +0x4000 (the commit) + * soc-east 0x502280000: SOC-East page, retention + east power words + * soc-ps 0x502800000: SOC page, the west axi2af power words + * wake 0x508004000: the S2R control + wake masks/status + * The handler refuses before touching any of them. + */ + reg = <0x5 0x02888000 0x0 0x20>, + <0x5 0x0836c000 0x0 0x4000>, + <0x5 0x02820000 0x0 0x20>, + <0x5 0x0282c000 0x0 0x1000>, + <0x5 0x00000000 0x0 0x80000>, + <0x5 0x09014000 0x0 0x1000>, + <0x5 0x09188000 0x0 0x1000>, + <0x5 0x08900000 0x0 0x8000>, + <0x5 0x02280000 0x0 0xd000>, + <0x5 0x02800000 0x0 0x9000>, + <0x5 0x08004000 0x0 0x1000>; + reg-names = "cpu-start", "wdt", "cluster", "pwrgate", + "clkgen", "spmi", "spmi-a0", "mastermini", + "soc-east", "soc-ps", "wake"; + /* + * ADT pmgr device-check-ps-b4-sleep (check-ps-b4-sleep = 0x1f): per + * display pipe its CPU and its last sub-block, 8 bytes apart. + */ + apple,check-ps-b4-sleep = <&pmgr_disp 0x0 0x58>, <&pmgr_aux 0x350 0x388>, + <&pmgr_aux 0x4a0 0x4d8>, <&pmgr_aux 0x4f0 0x528>, + <&pmgr_aux 0x2e8 0x320>; + apple,check-ps-b4-sleep-names = "disp", "dispext0", "dispext1", "dispext2", + "dispext3"; + /* + * apple,apcie, for the sleep of ANS's storage PCIe link (two ST lanes + * and a shared PHY), comes from the boot loader. From the ADT's + * /arm-io/apcie0 it adds it here and puts the axi2af, cio3pll, clkgen, + * phy-block and phy-ip windows and the apple,tunable-* arrays on the + * internal PCIe host (apple,t6020-pcie, the one with an "rc" + * reg-name), in the single-lane T8140 shape. It leaves a node that + * already has apple,apcie alone, so this tree must give neither + * apple,apcie nor an apcie-storage node. Without it a sleep with ANS + * stopped is refused, and suspend-to-idle stays the default. The ST + * power words (ps_ans, ps_apcie_st0/st1, ps_apcie_sys_st0/st1) are in + * SOC-East and are driven through the soc-east window. + */ + /* Only on a board whose sleep has been tested. */ + status = "disabled"; + }; }; &pmc { diff --git a/arch/arm64/boot/dts/apple/t604x.dtsi b/arch/arm64/boot/dts/apple/t604x.dtsi index 3484262fa7a43..a032ea8b3b07f 100644 --- a/arch/arm64/boot/dts/apple/t604x.dtsi +++ b/arch/arm64/boot/dts/apple/t604x.dtsi @@ -1370,6 +1370,10 @@ #gpio-cells = <2>; gpio-ranges = <&pinctrl 0 0 200>; apple,npins = <200>; + /* ADT gpio0: every pin is put back after a sleep, except no-restore-gpio-pins. */ + apple,resume-restore-all; + /* Pins the firmware lists as no-restore */ + gpio-reserved-ranges = <190 3>; interrupt-controller; #interrupt-cells = <2>; diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index 4ed50e0dc0b5c..6626229cbec35 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -66,6 +66,12 @@ struct spin_table_power_ops { #ifdef CONFIG_HOTPLUG_CPU int spin_table_set_power_ops(const struct spin_table_power_ops *ops); +#ifdef CONFIG_HIBERNATION +int spin_table_hibernate_pen_begin(phys_addr_t image_ttbr1, int primary); +void spin_table_hibernate_pen_end(bool frozen); +void spin_table_hibernate_restored(void); +bool spin_table_hibernate_cpus_stuck(void); +#endif int spin_table_hold_parked(void); void spin_table_unhold_parked(void); int spin_table_stop_parked(void); @@ -99,6 +105,19 @@ static inline bool spin_table_cpus_stuck(void) } #endif +#if !defined(CONFIG_HOTPLUG_CPU) || !defined(CONFIG_HIBERNATION) +static inline int spin_table_hibernate_pen_begin(phys_addr_t image_ttbr1, int primary) +{ + return 0; +} +static inline void spin_table_hibernate_pen_end(bool frozen) {} +static inline void spin_table_hibernate_restored(void) {} +static inline bool spin_table_hibernate_cpus_stuck(void) +{ + return false; +} +#endif + int __init init_cpu_ops(int cpu); extern const struct cpu_operations *get_cpu_ops(int cpu); diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h index e9ce68d50ba45..f453f11af781e 100644 --- a/arch/arm64/include/asm/suspend.h +++ b/arch/arm64/include/asm/suspend.h @@ -48,6 +48,38 @@ int swsusp_arch_resume(void); int arch_hibernation_header_save(void *addr, unsigned int max_size); int arch_hibernation_header_restore(void *addr); +/** + * struct arm64_hibernate_layout - What a platform adds to the boot layout check. + * @ex: Ranges that this boot keeps out of the image and that the layout check + * leaves out of the memory it compares, in ascending order. + * @nex: Number of valid entries in @ex. + * @have_base: Whether CPU release addresses are compared as offsets from @base. + * @base: Base address for the release addresses, if @have_base. + */ +struct arm64_hibernate_layout { + u64 ex[2][2]; + int nex; + bool have_base; + u64 base; +}; + +/** + * struct arm64_hibernate_platform - Platform hooks of arm64 hibernation. + * @pfn_nosave: Whether the platform keeps page frame @pfn out of the image. + * @layout: Mix the platform's boot facts into the layout cookie in @h with + * arm64_hibernate_layout_mix(), fill @lay, and return the new hash. The + * cookie is stored with an image and compared before it is restored: the + * boot that resumes an image must give the kernel the layout of the boot + * that wrote it. + */ +struct arm64_hibernate_platform { + bool (*pfn_nosave)(unsigned long pfn); + u64 (*layout)(u64 h, struct arm64_hibernate_layout *lay); +}; + +void arm64_hibernate_set_platform(const struct arm64_hibernate_platform *platform); +u64 arm64_hibernate_layout_mix(u64 h, const void *p, size_t len); + /* Used to resume on the CPU we hibernated on */ int hibernate_resume_nonboot_cpu_disable(void); diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index cac465111055b..263d682d48620 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -65,6 +65,9 @@ obj-$(CONFIG_ARM64_ACPI_PARKING_PROTOCOL) += acpi_parking_protocol.o obj-$(CONFIG_PARAVIRT) += paravirt.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o +ifdef CONFIG_HOTPLUG_CPU +obj-$(CONFIG_HIBERNATION) += smp_spin_table_pen.o +endif obj-$(CONFIG_ELF_CORE) += elfcore.o obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \ cpu-reset.o diff --git a/arch/arm64/kernel/hibernate-asm.S b/arch/arm64/kernel/hibernate-asm.S index 2baefe7a82d36..89fc1db49f618 100644 --- a/arch/arm64/kernel/hibernate-asm.S +++ b/arch/arm64/kernel/hibernate-asm.S @@ -67,19 +67,24 @@ SYM_CODE_START(swsusp_arch_suspend_exit) copy_page x0, x1, x2, x3, x4, x5, x6, x7, x8, x9 add x1, x10, #PAGE_SIZE - /* Clean the copied page to PoU - based on caches_clean_inval_pou() */ + /* + * Clean the copied page to PoC, not only PoU: non-coherent DMA + * masters (the coprocessors of some SoCs) read the restored pages + * from DRAM. Clean+invalidate to PoC also covers what + * caches_clean_inval_pou() did for the I-side. + */ raw_dcache_line_size x2, x3 sub x3, x2, #1 bic x4, x10, x3 2: /* clean D line / unified line */ -alternative_insn "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE + dc civac, x4 add x4, x4, x2 cmp x4, x1 b.lo 2b ldr x19, [x19, #HIBERN_PBE_NEXT] cbnz x19, 1b - dsb ish /* wait for PoU cleaning to finish */ + dsb ish /* wait for PoC cleaning to finish */ /* switch to the restored kernels page tables */ break_before_make_ttbr_switch x25, x21, x6, x8 diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 9398c8d0ecb03..ce251b5b20d0d 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -12,8 +12,11 @@ #define pr_fmt(x) "hibernate: " x #include #include +#include +#include #include #include +#include #include #include @@ -81,6 +84,9 @@ static struct arch_hibernate_hdr { phys_addr_t __hyp_stub_vectors; u64 sleep_cpu_mpidr; + + /* What the boot loader decided for the boot that wrote the image. */ + u64 boot_layout; } resume_hdr; static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i) @@ -89,13 +95,137 @@ static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i) memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version)); } +/** + * arm64_hibernate_layout_mix - Mix bytes into the boot layout cookie. + * @h: Hash so far. + * @p: Bytes to mix. + * @len: Number of bytes. + * + * FNV-1a, for the callback of a platform to mix its boot facts into the cookie + * in the same way as the rest of the layout. + */ +u64 arm64_hibernate_layout_mix(u64 h, const void *p, size_t len) +{ + const u8 *b = p; + + while (len--) { + h ^= *b++; + h *= 0x100000001b3ULL; /* FNV-1a */ + } + return h; +} + +/* + * Mix one memory span less the parts in @ex, ranges that this boot keeps out of + * the image. iBoot's no-map allocations inside aura's fixed layout (the DCP + * oslogs, ANS firmware) move with the boot object and split memblock's regions + * differently on every boot; what lies there never holds an image page. + */ +static u64 hibernate_mix_span(u64 h, u64 b, u64 e, u32 flags, const u64 (*ex)[2], int nex) +{ + u64 size; + + if (b >= e) + return h; + if (nex && e > ex[0][0] && b < ex[0][1]) { + h = hibernate_mix_span(h, b, ex[0][0], flags, ex + 1, nex - 1); + return hibernate_mix_span(h, ex[0][1], e, flags, ex + 1, nex - 1); + } + if (nex) + return hibernate_mix_span(h, b, e, flags, ex + 1, nex - 1); + size = e - b; + h = arm64_hibernate_layout_mix(h, &b, sizeof(b)); + h = arm64_hibernate_layout_mix(h, &size, sizeof(size)); + return arm64_hibernate_layout_mix(h, &flags, sizeof(flags)); +} + +static const struct arm64_hibernate_platform *hibernate_platform __ro_after_init; + +/** + * arm64_hibernate_set_platform - Register the platform's hibernation hooks. + * @platform: Hooks of the platform; they must stay valid. + * + * Only one platform can register, at boot. + */ +void __init arm64_hibernate_set_platform(const struct arm64_hibernate_platform *platform) +{ + if (WARN_ON_ONCE(hibernate_platform)) + return; + hibernate_platform = platform; +} + +static bool hibernate_platform_pfn_nosave(unsigned long pfn) +{ + return hibernate_platform && hibernate_platform->pfn_nosave && + hibernate_platform->pfn_nosave(pfn); +} + +/* + * The image restores every page to the frame it came from, so the boot that + * resumes it must give the kernel what the boot that wrote it did: the same + * RAM, the same CPUs started the same way, and whatever fixed layout the + * platform's boot loader gives the kernel, which the platform's hook adds. A + * boot loader decides them on every boot; the kernel build check above cannot + * see them. + */ +static u64 hibernate_boot_layout(void) +{ + struct arm64_hibernate_layout lay = {}; + u64 h = 0xcbf29ce484222325ULL, rel, hwid; + struct memblock_region *r; + struct device_node *np; + u8 kept; + + if (hibernate_platform && hibernate_platform->layout) + h = hibernate_platform->layout(h, &lay); + + for_each_of_cpu_node(np) { + if (of_property_read_u64(np, "cpu-release-addr", &rel)) + continue; + hwid = of_get_cpu_hwid(np, 0); + kept = hibernate_platform_pfn_nosave(PHYS_PFN(rel)); + if (lay.have_base) + rel -= lay.base; + h = arm64_hibernate_layout_mix(h, &hwid, sizeof(hwid)); + h = arm64_hibernate_layout_mix(h, &rel, sizeof(rel)); + h = arm64_hibernate_layout_mix(h, &kept, sizeof(kept)); + } + + for_each_mem_region(r) + h = hibernate_mix_span(h, r->base, r->base + r->size, r->flags, lay.ex, lay.nex); + return h; +} + +/* + * Most image pages are loaded straight into their original frames through the + * cacheable linear map. Non-coherent DMA masters (on Apple SoCs: the GPU page + * table walker, DCP, every RTKit co-processor, the NVMe queues) read DRAM, and + * a later eviction of a dirty line would overwrite what a device wrote since. + * Clean and invalidate the page to the PoC before the jump. + * + * A frame that is not in the boot kernel's linear map (for example the linear + * alias of the kernel image, which faults when touched) had nothing loaded + * through the linear map: such frames are restored by the restore_pblist copy, + * which cleans each copied page itself. + */ +int arch_hibernate_clean_image_page(unsigned long pfn) +{ + unsigned long va; + + if (!kernel_page_present(pfn_to_page(pfn))) + return 0; + va = (unsigned long)page_address(pfn_to_page(pfn)); + dcache_clean_inval_poc(va, va + PAGE_SIZE); + return 1; +} + int pfn_is_nosave(unsigned long pfn) { unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin); unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1); return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn)) || - crash_is_nosave(pfn); + crash_is_nosave(pfn) || hibernate_platform_pfn_nosave(pfn); } void notrace save_processor_state(void) @@ -129,8 +259,9 @@ int arch_hibernation_header_save(void *addr, unsigned int max_size) return -ENODEV; } hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu); - pr_info("Hibernating on CPU %d [mpidr:0x%llx]\n", sleep_cpu, - hdr->sleep_cpu_mpidr); + hdr->boot_layout = hibernate_boot_layout(); + pr_info("Hibernating on CPU %d [mpidr:0x%llx], boot layout %016llx\n", sleep_cpu, + hdr->sleep_cpu_mpidr, hdr->boot_layout); return 0; } @@ -138,6 +269,7 @@ EXPORT_SYMBOL(arch_hibernation_header_save); int arch_hibernation_header_restore(void *addr) { + u64 layout; int ret; struct arch_hibernate_hdr_invariants invariants; struct arch_hibernate_hdr *hdr = addr; @@ -147,6 +279,12 @@ int arch_hibernation_header_restore(void *addr) pr_crit("Hibernate image not generated by this kernel!\n"); return -EINVAL; } + layout = hibernate_boot_layout(); + if (hdr->boot_layout != layout) { + pr_crit("Hibernate image written under another boot layout (%016llx, this boot %016llx): not restored\n", + hdr->boot_layout, layout); + return -EINVAL; + } sleep_cpu = get_logical_index(hdr->sleep_cpu_mpidr); pr_info("Hibernated on CPU %d [mpidr:0x%llx]\n", sleep_cpu, @@ -337,7 +475,7 @@ int swsusp_arch_suspend(void) unsigned long flags; struct sleep_stack_data state; - if (cpus_are_stuck_in_kernel() || spin_table_cpus_stuck()) { + if (cpus_are_stuck_in_kernel() || spin_table_hibernate_cpus_stuck()) { pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n"); return -EBUSY; } @@ -386,6 +524,9 @@ int swsusp_arch_suspend(void) sleep_cpu = -EINVAL; __cpu_suspend_exit(); + /* The boot kernel's secondaries are in its pen, not where the image parked them. */ + spin_table_hibernate_restored(); + /* * Just in case the boot kernel did turn the SSBD * mitigation off behind our back, let's set the state @@ -489,10 +630,19 @@ int __nocfi swsusp_arch_resume(void) int hibernate_resume_nonboot_cpu_disable(void) { + int ret; + if (sleep_cpu < 0) { pr_err("Failing to resume from hibernate on an unknown CPU.\n"); return -ENODEV; } - return freeze_secondary_cpus(sleep_cpu); + /* Parking spin-table CPUs wait outside the image while it is restored. */ + ret = spin_table_hibernate_pen_begin(resume_hdr.ttbr1_el1, sleep_cpu); + if (ret) + return ret; + ret = freeze_secondary_cpus(sleep_cpu); + spin_table_hibernate_pen_end(!ret); + + return ret; } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index f6b5decff53da..02badcc245d05 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -1354,14 +1354,29 @@ bool smp_crash_stop_failed(void) } #endif +/* + * The CPUs that have to leave the kernel are the others. A boot CPU whose node + * carries no enable-method (Apple boot objects describe only the secondaries' + * spin table) has no operations of its own: then every other possible CPU must + * have a way out. + */ static bool have_cpu_die(void) { #ifdef CONFIG_HOTPLUG_CPU - int any_cpu = raw_smp_processor_id(); - const struct cpu_operations *ops = get_cpu_ops(any_cpu); + int this_cpu = raw_smp_processor_id(), cpu; + const struct cpu_operations *ops = get_cpu_ops(this_cpu); - if (ops && ops->cpu_die) - return true; + if (ops) + return ops->cpu_die != NULL; + + for_each_possible_cpu(cpu) { + if (cpu == this_cpu) + continue; + ops = get_cpu_ops(cpu); + if (!ops || !ops->cpu_die) + return false; + } + return num_possible_cpus() > 1; #endif return false; } diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 2d7660bd22f80..9d5f26afe4edd 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -5,10 +5,12 @@ * Copyright (C) 2013 ARM Ltd. */ +#include #include #include #include #include +#include #include #include @@ -144,8 +146,10 @@ const struct cpu_operations smp_spin_table_ops = { * masked. From there it goes back through secondary_holding_pen, or, when the * platform suspends the system, it stops and takes its last WFI after the commit. * It is then started again at its reset vector. Stopping and starting a CPU is - * SoC-specific, so a platform driver provides it. Until one has, CPU hotplug - * fails cleanly, as on plain spin-table. + * SoC-specific, so a platform driver provides it (spin_table_set_power_ops()); + * until one has, a suspend that needs the parked CPUs stopped is refused. + * Parking and leaving the park need only cpu_soft_restart(): without + * KEXEC_CORE, CPU hotplug fails cleanly, as on plain spin-table. * * A CPU left awake at the commit can hang the machine, and the commit can't be * undone. So each parked CPU first answers a hold, while the suspend can still @@ -163,6 +167,198 @@ static bool spin_table_stop_released; #define SPIN_TABLE_WAIT_US 100000 +#if defined(CONFIG_HIBERNATION) && defined(CONFIG_KEXEC_CORE) +/* + * Resuming from hibernation, the boot kernel takes its secondaries offline and + * then overwrites every page the image saved, the park and its cpumasks + * included. A CPU parked there would run on in memory that changes under it, + * and its translation tables with it. On that path a CPU leaving the kernel + * waits instead in a pen the image doesn't contain: the code and a slot per + * CPU in .data..nosave, run with the MMU off. arm64 resumes only an image of + * the kernel build that is running; with the boot kernel loaded at the image + * kernel's physical address (checked when the resume starts) the pen is at the + * same place in both, and the restored kernel reads what the boot kernel left + * there. It starts each CPU from the pen through secondary_holding_pen, as at + * boot. Nothing is powered off, so this needs no platform power ops. + */ +#define SPIN_TABLE_PEN_MAGIC 0x314e455034535053ULL /* "SPS4PEN1" */ +#define SPIN_TABLE_PEN_HERE 0x45524548ULL /* "HERE", from the pen */ + +struct spin_table_pen_slot { + u64 release; /* where the CPU goes; 0 holds it */ + u64 here; /* the CPU reached the pen */ + u64 mpidr; +} __aligned(ARCH_DMA_MINALIGN); + +static struct { + u64 magic; + u64 self; /* this record's address in the kernel that wrote it */ + unsigned long penned[BITS_TO_LONGS(NR_CPUS)]; + struct spin_table_pen_slot slot[NR_CPUS]; +} spin_table_pen __nosavedata __aligned(ARCH_DMA_MINALIGN); + +extern char spin_table_hibernate_pen[]; +extern const struct cpu_operations smp_rvbar_spin_table_ops; +static struct cpumask spin_table_cpus_penned; +static bool spin_table_pen_requested; + +static void spin_table_pen_clean(void *start, size_t size) +{ + dcache_clean_inval_poc((unsigned long)start, (unsigned long)start + size); +} + +/* + * Resuming from hibernation, before the boot kernel takes its secondaries offline. + * The restored kernel finds the pen only where it is in its own image, which holds + * when the boot kernel is loaded at the image kernel's physical address. arm64 can + * restore from a boot kernel placed elsewhere (@image_ttbr1, the image's + * swapper_pg_dir, is used for the jump), but there the pen's frames are image + * frames: the restore would overwrite them while the penned CPUs run from them with + * the MMU off. Refuse the resume then, as long as a CPU other than @primary (the + * one that stays online) would need the pen. + */ +int spin_table_hibernate_pen_begin(phys_addr_t image_ttbr1, int primary) +{ + phys_addr_t here = __pa_symbol(swapper_pg_dir); + int cpu; + + /* No record of an earlier, abandoned resume may reach the restored kernel. */ + spin_table_pen.magic = 0; + spin_table_pen_clean(&spin_table_pen, sizeof(spin_table_pen.magic)); + + if (image_ttbr1 != here) { + for_each_online_cpu(cpu) { + if (cpu == primary || get_cpu_ops(cpu) != &smp_rvbar_spin_table_ops) + continue; + pr_err("hibernation: the image's kernel ran at another physical address (swapper_pg_dir %pa, here %pa): spin-table CPUs cannot wait in this kernel's pen, resume refused\n", + &image_ttbr1, &here); + return -EINVAL; + } + return 0; + } + WRITE_ONCE(spin_table_pen_requested, true); + return 0; +} + +/* + * After they went offline. With all of them frozen, the restored kernel finds the + * record; otherwise the resume is abandoned and they go back online from the pen. + */ +void spin_table_hibernate_pen_end(bool frozen) +{ + unsigned int cpu; + + WRITE_ONCE(spin_table_pen_requested, false); + if (!frozen || cpumask_empty(&spin_table_cpus_penned)) + return; + + bitmap_zero(spin_table_pen.penned, NR_CPUS); + for_each_cpu(cpu, &spin_table_cpus_penned) + __set_bit(cpu, spin_table_pen.penned); + spin_table_pen.self = __pa_symbol(&spin_table_pen); + spin_table_pen.magic = SPIN_TABLE_PEN_MAGIC; + spin_table_pen_clean(&spin_table_pen, offsetof(typeof(spin_table_pen), slot)); + pr_info("hibernation: %u CPUs wait outside the image\n", + cpumask_weight(&spin_table_cpus_penned)); +} + +/* + * In the restored kernel, right after the image took over: the CPUs the image parked + * before its snapshot are in the boot kernel's pen now. + */ +void spin_table_hibernate_restored(void) +{ + unsigned int cpu, found = 0; + + if (spin_table_pen.magic != SPIN_TABLE_PEN_MAGIC) + return; + spin_table_pen.magic = 0; + spin_table_pen_clean(&spin_table_pen, sizeof(spin_table_pen.magic)); + if (spin_table_pen.self != __pa_symbol(&spin_table_pen)) { + pr_err("hibernation: the boot kernel's CPU pen was at %#llx, not here: its CPUs stay offline\n", + spin_table_pen.self); + return; + } + + cpumask_clear(&spin_table_cpus_penned); + for_each_present_cpu(cpu) { + if (!test_bit(cpu, spin_table_pen.penned)) + continue; + if (spin_table_pen.slot[cpu].mpidr != cpu_logical_map(cpu)) { + pr_err("hibernation: CPU%u: the pen holds MPIDR %#llx, not %#llx\n", cpu, + spin_table_pen.slot[cpu].mpidr, cpu_logical_map(cpu)); + continue; + } + cpumask_clear_cpu(cpu, &spin_table_cpus_parked); + cpumask_clear_cpu(cpu, &spin_table_cpus_held); + cpumask_clear_cpu(cpu, &spin_table_cpus_unpark); + cpumask_clear_cpu(cpu, &spin_table_cpus_off); + cpumask_set_cpu(cpu, &spin_table_cpus_penned); + found++; + } + pr_info("hibernation: %u CPUs start from the boot kernel's pen\n", found); +} + +/* Leaves the kernel for the pen; the boot CPU waits for it to arrive (cpu_kill). */ +static void __noreturn smp_rvbar_spin_table_pen(unsigned int cpu, u64 mpidr) +{ + struct spin_table_pen_slot *slot = &spin_table_pen.slot[cpu]; + typeof(cpu_soft_restart) *restart; + + slot->release = 0; + slot->here = 0; + slot->mpidr = mpidr; + spin_table_pen_clean(slot, sizeof(*slot)); + cpumask_set_cpu(cpu, &spin_table_cpus_penned); + + cpu_install_idmap(); + restart = (void *)__pa_symbol(cpu_soft_restart); + restart(0, __pa_symbol(spin_table_hibernate_pen), __pa_symbol(slot), 0, 0); + unreachable(); +} + +static bool spin_table_pen_arrived(unsigned int cpu) +{ + struct spin_table_pen_slot *slot = &spin_table_pen.slot[cpu]; + + /* Written with the MMU off: drop any copy of the line before reading. */ + dcache_inval_poc((unsigned long)slot, (unsigned long)(slot + 1)); + return READ_ONCE(slot->here) == SPIN_TABLE_PEN_HERE; +} + +/* Starts a CPU waiting in the pen; it goes on to secondary_holding_pen. */ +static void spin_table_pen_release(unsigned int cpu) +{ + struct spin_table_pen_slot *slot = &spin_table_pen.slot[cpu]; + + WRITE_ONCE(slot->release, __pa_symbol(secondary_holding_pen)); + spin_table_pen_clean(slot, sizeof(*slot)); +} +#else +static const bool spin_table_pen_requested; +static struct cpumask spin_table_cpus_penned; + +static void smp_rvbar_spin_table_pen(unsigned int cpu, u64 mpidr) +{ +} + +static bool spin_table_pen_arrived(unsigned int cpu) +{ + return false; +} + +static void spin_table_pen_release(unsigned int cpu) +{ +} + +#ifdef CONFIG_HIBERNATION +/* Without cpu_soft_restart() a CPU cannot leave the kernel: nothing to keep. */ +int spin_table_hibernate_pen_begin(phys_addr_t image_ttbr1, int primary) { return 0; } +void spin_table_hibernate_pen_end(bool frozen) {} +void spin_table_hibernate_restored(void) {} +#endif +#endif + int spin_table_set_power_ops(const struct spin_table_power_ops *ops) { return cmpxchg(&spin_table_power, NULL, ops) ? -EBUSY : 0; @@ -208,6 +404,9 @@ int spin_table_hold_parked(void) { int us; + if (!READ_ONCE(spin_table_power)) + return -EOPNOTSUPP; + WRITE_ONCE(spin_table_hold_requested, true); dsb(ish); sev(); @@ -278,7 +477,9 @@ static int smp_rvbar_spin_table_cpu_boot(unsigned int cpu) /* The CPU may run straight into the pen: release it before it starts. */ write_pen_release(mpidr); - if (cpumask_test_and_clear_cpu(cpu, &spin_table_cpus_off)) { + if (cpumask_test_and_clear_cpu(cpu, &spin_table_cpus_penned)) { + spin_table_pen_release(cpu); + } else if (cpumask_test_and_clear_cpu(cpu, &spin_table_cpus_off)) { ret = spin_table_power->cpu_on(mpidr); if (ret) { cpumask_set_cpu(cpu, &spin_table_cpus_off); @@ -301,10 +502,11 @@ static bool smp_rvbar_spin_table_cpu_can_disable(unsigned int cpu) static int smp_rvbar_spin_table_cpu_disable(unsigned int cpu) { - if (!IS_ENABLED(CONFIG_KEXEC_CORE)) - return -EOPNOTSUPP; - - return READ_ONCE(spin_table_power) ? 0 : -EOPNOTSUPP; + /* + * A parked CPU comes back through cpu_soft_restart(). Only stopping it for a + * suspend needs the platform's power ops (spin_table_hold_parked()). + */ + return IS_ENABLED(CONFIG_KEXEC_CORE) ? 0 : -EOPNOTSUPP; } #ifdef CONFIG_KEXEC_CORE @@ -350,11 +552,14 @@ static void smp_rvbar_spin_table_cpu_die(unsigned int cpu) u64 mpidr = cpu_logical_map(cpu); /* The early-boot and crash paths get here without hotplug support. */ - if (!READ_ONCE(spin_table_power) || !IS_ENABLED(CONFIG_KEXEC_CORE)) { + if (!IS_ENABLED(CONFIG_KEXEC_CORE)) { while (1) wfe(); } + if (READ_ONCE(spin_table_pen_requested)) + smp_rvbar_spin_table_pen(cpu, mpidr); + cpumask_set_cpu(cpu, &spin_table_cpus_parked); while (!cpumask_test_cpu(cpu, &spin_table_cpus_unpark)) { @@ -373,12 +578,11 @@ static int smp_rvbar_spin_table_cpu_kill(unsigned int cpu) { int us; - if (!READ_ONCE(spin_table_power)) - return 0; - for (us = 0; us < SPIN_TABLE_WAIT_US; us += 10) { if (cpumask_test_cpu(cpu, &spin_table_cpus_parked)) return 0; + if (cpumask_test_cpu(cpu, &spin_table_cpus_penned) && spin_table_pen_arrived(cpu)) + return 0; udelay(10); } @@ -409,4 +613,17 @@ bool spin_table_cpus_stuck(void) return true; return false; } + +#ifdef CONFIG_HIBERNATION +/* + * For a hibernation that only holds without the pen. With it the kernel that + * resumes an image takes its offlined CPUs out of the memory the restore + * overwrites before it starts (spin_table_hibernate_pen_begin()), and the + * restored kernel starts them from there. + */ +bool spin_table_hibernate_cpus_stuck(void) +{ + return !IS_ENABLED(CONFIG_KEXEC_CORE) && spin_table_cpus_stuck(); +} +#endif #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/arm64/kernel/smp_spin_table_pen.S b/arch/arm64/kernel/smp_spin_table_pen.S new file mode 100644 index 0000000000000..4610bc09ad002 --- /dev/null +++ b/arch/arm64/kernel/smp_spin_table_pen.S @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Where a parking spin-table CPU waits while the kernel resuming from + * hibernation overwrites memory with the image (smp_spin_table.c). + */ + +#include +#include + +/* + * x0: the CPU's struct spin_table_pen_slot: the entry address at offset 0, + * written by the kernel that starts the CPU again, and the arrival word at 8, + * written here. + * + * Entered from cpu_soft_restart() with the MMU and caches off and interrupts + * masked, so nothing here depends on the kernel's page tables or on cache + * maintenance by this CPU. The code and the slots are in .data..nosave: the + * image never contains them, so the restore leaves both as the boot kernel + * wrote them, at the same physical address in the restored kernel. + */ + .pushsection ".data..nosave", "aw" + .balign 64 +SYM_CODE_START(spin_table_hibernate_pen) + movz x1, #0x4548 + movk x1, #0x4552, lsl #16 // "HERE" + str x1, [x0, #8] + dsb sy +1: ldr x1, [x0] + cbnz x1, 2f + wfe + b 1b +2: br x1 +SYM_CODE_END(spin_table_hibernate_pen) + .popsection diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 6ac3321f4c575..0773f1c1209e8 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -1436,6 +1437,58 @@ static void kvm_timer_handle_errata(void) } } +/* + * A guest's counters are the physical counter minus an offset that is fixed + * when the VM is made. The physical counter can start again in a sleep: it + * always does across a resume from disk, which is a cold boot. The guest's + * counters then fall back by what the counter lost, and the guest's clock + * stands still until the counter has caught up, which takes as long as the + * host had been up. Move the offsets by the same amount instead: the + * guest's counters go on from the value they had when the system went down, + * as for a VM that was stopped that long. A counter that keeps running + * through the sleep is left alone, and the guest sees the sleep as time that + * passed. + * + * Syscore: one CPU, interrupts off, every task frozen, so no vCPU is loaded + * and nothing changes vm_list. + */ +static u64 kvm_timer_sleep_cnt; + +static int kvm_timer_syscore_suspend(void *data) +{ + kvm_timer_sleep_cnt = kvm_phys_timer_read(); + return 0; +} + +static void kvm_timer_syscore_resume(void *data) +{ + u64 now = kvm_phys_timer_read(); + unsigned int vms = 0; + struct kvm *kvm; + u64 lost; + + if (now >= kvm_timer_sleep_cnt) + return; + lost = kvm_timer_sleep_cnt - now; + list_for_each_entry(kvm, &vm_list, vm_list) { + kvm->arch.timer_data.voffset -= lost; + kvm->arch.timer_data.poffset -= lost; + vms++; + } + if (vms) + kvm_info("the counter started again in the sleep (%llu -> %llu): the counters of %u VM(s) kept\n", + kvm_timer_sleep_cnt, now, vms); +} + +static const struct syscore_ops kvm_timer_syscore_ops = { + .suspend = kvm_timer_syscore_suspend, + .resume = kvm_timer_syscore_resume, +}; + +static struct syscore kvm_timer_syscore = { + .ops = &kvm_timer_syscore_ops, +}; + int __init kvm_timer_hyp_init(bool has_gic) { struct arch_timer_kvm_info *info; @@ -1505,6 +1558,7 @@ int __init kvm_timer_hyp_init(bool has_gic) } kvm_timer_handle_errata(); + register_syscore(&kvm_timer_syscore); return 0; out_free_ptimer_irq: diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 379fa031ce599..c989aa9eb3f97 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -1059,6 +1059,14 @@ static void apple_fast_die_sample(struct apple_fast_die *fd) fd->waiting = false; } else { limit = fd->fallback; + /* + * The PMP stopped after it had run (a resume from disk restarts it): + * the wait, and its warning, start over from now. + */ + if (!fd->waiting && fd->report) { + fd->start = jiffies; + fd->warned = false; + } fd->waiting = true; if (!fd->warned && time_after(jiffies, fd->start + APPLE_FAST_DIE_WARN_S * HZ)) { diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index 330bb7fa6b753..a27b50591a610 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -645,6 +645,7 @@ static const struct sio_hw_data sio_hw_t6030 = { * the 5 of the host's mailbox call timeout is not part of the record. */ static const struct sio_hw_data sio_hw_t6041 = { + .stop_at_commit = true, .protocol = 10, .fifo_threshold = 0x600, .cfg_word4 = 0x800, @@ -1838,6 +1839,20 @@ static int sio_send_v9_shmem(struct sio_data *sio, return 0; } +/* Publish the shared memory to the firmware; also used after a cold restart. */ +static int sio_send_shmem_params(struct sio_data *sio) +{ + int err; + + if (sio->protocol == 9) + return sio_send_v9_shmem(sio, sio_set_param); + + err = sio_set_param(sio, 1, sio->shmem_iova >> 12); + if (err) + return err; + return sio_set_param(sio, 2, SIO_SHMEM_SIZE); +} + static int sio_alloc_shmem(struct sio_data *sio) { const struct sio_dma_arrays *arrays = sio->hw->arrays; @@ -1879,13 +1894,8 @@ static int sio_alloc_shmem(struct sio_data *sio) err = devm_add_action_or_reset(sio->dev, sio_free_v9_shmem, sio); if (err) return err; - return sio_send_v9_shmem(sio, sio_set_param); } - - err = sio_set_param(sio, 1, sio->shmem_iova >> 12); - if (err) - return err; - return sio_set_param(sio, 2, SIO_SHMEM_SIZE); + return sio_send_shmem_params(sio); free_shmem: /* No address has been published yet. */ @@ -2141,9 +2151,15 @@ static bool sio_chan_busy(struct sio_chan *siochan) static __maybe_unused int sio_suspend_late(struct device *dev) { struct sio_data *sio = dev_get_drvdata(dev); + bool dry; int i, ret; - if (!pm_suspend_via_firmware() || sio->asleep) + /* + * A dry run keeps the power, and an idled firmware doesn't wake with it kept. One whose + * stop is the commit's is ready there as for a real sleep: a dry run stops nothing. + */ + dry = !apple_sleep_power_loss() && sio->commit_registered && pm_suspend_via_firmware(); + if ((!apple_sleep_power_loss() && !dry) || sio->asleep) return 0; for (i = 0; i < sio->nchannels; i++) { @@ -2153,7 +2169,7 @@ static __maybe_unused int sio_suspend_late(struct device *dev) } } - if (sio->commit_registered && apple_sleep_commit_stops_coprocs()) { + if (sio->commit_registered && (dry || apple_sleep_commit_stops_coprocs())) { apple_sleep_coproc_set_ready(dev, true); sio->commit_used = true; sio->asleep = true; @@ -2178,6 +2194,70 @@ static __maybe_unused int sio_suspend_late(struct device *dev) return 0; } +/* + * Bring a freshly loaded, halted SIO up again under the existing host state: + * the firmware-facing part of probe, reusing the shared memory, the DMA + * channels and their clients. + */ +static int sio_restart(struct sio_data *sio) +{ + u32 protocol = sio->protocol; + int err, i; + + /* Nothing can answer the calls that were in flight. */ + for (i = 1; i < SIO_NTAGS; i++) { + if (test_and_clear_bit(i, sio->tags.allocated)) + dev_warn(sio->dev, "dropping stale call tag %d\n", i); + reinit_completion(&sio->tags.completions[i]); + } + + err = apple_rtkit_reinit(sio->rtk); + if (err) { + dev_err(sio->dev, "restart: RTKit reinit: %d\n", err); + return err; + } + + writel(CPU_CONTROL_RUN, sio->base + REG_CPU_CONTROL); + + err = apple_rtkit_boot(sio->rtk); + if (err) { + dev_err(sio->dev, "restart: SIO did not boot: %d\n", err); + return err; + } + + err = apple_rtkit_start_ep(sio->rtk, EP_SIO); + if (err) { + dev_err(sio->dev, "restart: starting SIO endpoint: %d\n", err); + return err; + } + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_START)); + if (err < 1) { + err = err ? err : -ENXIO; + dev_err(sio->dev, "restart: starting SIO service: %d\n", err); + return err; + } + if (sio->protocol != protocol) { + dev_err(sio->dev, "restart: protocol changed from v%u to v%u\n", + protocol, sio->protocol); + return -EPROTO; + } + + err = sio_send_dt_params(sio); + if (err < 0) { + dev_err(sio->dev, "restart: boot-up parameters: %d\n", err); + return err; + } + + err = sio_send_shmem_params(sio); + if (err < 0) { + dev_err(sio->dev, "restart: shared memory: %d\n", err); + return err; + } + + return 0; +} + static __maybe_unused int sio_resume_early(struct device *dev) { struct sio_data *sio = dev_get_drvdata(dev); @@ -2232,8 +2312,40 @@ static __maybe_unused int sio_resume_early(struct device *dev) return 0; } +static __maybe_unused int sio_restore(struct device *dev) +{ + struct sio_data *sio = dev_get_drvdata(dev); + int err; + + /* + * The resume boot loaded the SIO firmware again and left the CPU + * halted (this driver is not part of the resume initramfs), while the + * restored session still believes the old instance runs. Start the + * new instance; DMA clients set their channels up again when they + * reprepare their streams. + */ + if (readl(sio->base + REG_CPU_CONTROL) & CPU_CONTROL_RUN) { + dev_dbg(dev, "resume from disk: SIO still running, session kept\n"); + } else { + err = sio_restart(sio); + if (!err) + dev_dbg(dev, "resume from disk: SIO restarted (protocol v%u)\n", + sio->protocol); + else + dev_err(dev, "resume from disk: SIO restart failed: %d\n", err); + } + + return pm_runtime_force_resume(dev); +} + +/* A firmware-driven suspend stops the firmware late; a resume from disk restarts it. */ static const struct dev_pm_ops sio_pm_ops = { - SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) + .suspend = pm_sleep_ptr(pm_runtime_force_suspend), + .resume = pm_sleep_ptr(pm_runtime_force_resume), + .freeze = pm_sleep_ptr(pm_runtime_force_suspend), + .thaw = pm_sleep_ptr(pm_runtime_force_resume), + .poweroff = pm_sleep_ptr(pm_runtime_force_suspend), + .restore = pm_sleep_ptr(sio_restore), LATE_SYSTEM_SLEEP_PM_OPS(sio_suspend_late, sio_resume_early) RUNTIME_PM_OPS(sio_suspend, sio_resume, NULL) }; diff --git a/drivers/gpu/drm/asahi/g17/sleep.c b/drivers/gpu/drm/asahi/g17/sleep.c index 2aebc75d2188f..e3e0d9c3c654f 100644 --- a/drivers/gpu/drm/asahi/g17/sleep.c +++ b/drivers/gpu/drm/asahi/g17/sleep.c @@ -322,7 +322,7 @@ struct asahi_g17_commit *asahi_g17_commit_prepare(struct device *dev, if (c->request) apple_pmp_report_entry_commit_owned(c->request, true); - ret = apple_sleep_coproc_register_first(dev, asahi_g17_commit_stop, c); + ret = apple_sleep_coproc_register_first(dev, APPLE_SLEEP_FIRST_GPU, asahi_g17_commit_stop, c); if (ret) goto err; apple_sleep_coproc_set_ready(dev, true); @@ -369,7 +369,7 @@ struct asahi_g17_commit *asahi_g16_commit_prepare(struct device *dev, struct app if (!c->cpu[0] || !c->ps) goto err; - ret = apple_sleep_coproc_register_first(dev, asahi_g17_commit_stop, c); + ret = apple_sleep_coproc_register_first(dev, APPLE_SLEEP_FIRST_GPU, asahi_g17_commit_stop, c); if (ret) goto err; apple_sleep_coproc_set_ready(dev, true); diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 57a5f868a3606..a94029884eddb 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -349,6 +349,18 @@ static const struct aic_info aic3_t8132_info __initconst = { .fw_suspend = true, }; +/* + * The same AICv3 as T8140's across a firmware-driven suspend: the CONFIG, + * TIMEOUT and CLUSTER_PRIO words and the IRQ_CFG array are lost with its power. + */ +static const struct aic_info aic3_t6041_info __initconst = { + .version = 3, + .irq_cfg = AIC3_IRQ_CFG, + .fast_ipi = true, + .local_fast_ipi = true, + .fw_suspend = true, +}; + static const struct aic_info aic3_t8142_info __initconst = { .version = 3, .irq_cfg = AIC3_IRQ_CFG, @@ -366,6 +378,10 @@ static const struct of_device_id aic_info_match[] = { .compatible = "apple,t8140-aic3", .data = &aic3_t8140_info, }, + { + .compatible = "apple,t6041-aic3", + .data = &aic3_t6041_info, + }, { .compatible = "apple,t8142-aic3", .data = &aic3_t8142_info, @@ -1165,7 +1181,8 @@ static int aic_syscore_suspend(void *data) struct aic_irq_chip *ic = aic_irqc; u32 die, i, off = 0; - if (!pm_suspend_via_firmware()) + /* Registered for the resume from disk alone where the SoC has no such suspend. */ + if (!ic->info.fw_suspend || !pm_suspend_via_firmware()) return 0; aic_sleep.config = aic_ic_read(ic, AIC2_CONFIG); @@ -1205,13 +1222,45 @@ static void aic_resume_boot_cpu(void) } } +/* + * After a resume from disk the controller holds what the boot kernel left: its + * init masked every IRQ and it unmasked only the ones it used itself. The IRQ + * core disables lazily and unmasks only an IRQ it recorded as masked, so on + * its own it never unmasks the restored kernel's other IRQs again. Make every + * hardware mask match the restored descriptor: unmasked unless the core + * recorded it masked (a lazily disabled IRQ was unmasked in hardware when the + * image was taken), and masked when this kernel has no mapping for it. + */ +static void aic_resync_masks(struct aic_irq_chip *ic) +{ + unsigned int die, i; + + for (die = 0; die < ic->nr_die; die++) { + for (i = 0; i < ic->nr_irq; i++) { + irq_hw_number_t hwirq = AIC_IRQ_HWIRQ(die, i); + unsigned int virq = irq_find_mapping(ic->hw_domain, hwirq); + struct irq_data *d = virq ? irq_get_irq_data(virq) : NULL; + u32 off = die * ic->info.die_stride; + + if (d && !irqd_irq_masked(d)) + aic_ic_write(ic, ic->info.mask_clr + off + MASK_REG(i), MASK_BIT(i)); + else + aic_ic_write(ic, ic->info.mask_set + off + MASK_REG(i), MASK_BIT(i)); + } + } +} + static void aic_syscore_resume(void *data) { struct aic_irq_chip *ic = aic_irqc; u32 die, i, off = 0; - if (!aic_sleep.saved) + if (!aic_sleep.saved) { + /* No firmware-driven suspend: only a resume from disk has lost the masks. */ + if (pm_hibernate_is_restored()) + aic_resync_masks(ic); return; + } aic_sleep.saved = false; /* Otherwise the suspend was refused, and the controller stayed powered. */ @@ -1254,14 +1303,21 @@ static struct syscore aic_syscore = { .ops = &aic_syscore_ops, }; +/* + * The save across a firmware-driven suspend is for the SoCs that have one. The + * mask resync after a resume from disk is for every SoC: whatever the AIC, the + * kernel that loaded the image left its own masks in it. + */ static void __init aic_sleep_init(struct aic_irq_chip *irqc) { - if (!irqc->info.fw_suspend) + if (!irqc->info.fw_suspend && !IS_ENABLED(CONFIG_HIBERNATION)) return; - aic_sleep.irq_cfg = kcalloc(irqc->nr_die * irqc->nr_irq, sizeof(u32), GFP_KERNEL); - if (WARN_ON(!aic_sleep.irq_cfg)) - return; + if (irqc->info.fw_suspend) { + aic_sleep.irq_cfg = kcalloc(irqc->nr_die * irqc->nr_irq, sizeof(u32), GFP_KERNEL); + if (WARN_ON(!aic_sleep.irq_cfg)) + return; + } register_syscore(&aic_syscore); } diff --git a/drivers/mfd/macsmc-test.c b/drivers/mfd/macsmc-test.c index a2dc3531ac36f..29920e5c482c8 100644 --- a/drivers/mfd/macsmc-test.c +++ b/drivers/mfd/macsmc-test.c @@ -10,9 +10,10 @@ struct macsmc_test_context { unsigned int calls; bool require_sram_lock; /* What the sleep tests saw, in order: keys read, values written, notices. */ - u32 seen[8]; + u32 seen[12]; unsigned int seen_count; int ap_power_result; + int atomic_result; u64 record; }; @@ -26,9 +27,11 @@ static int macsmc_test_command(struct apple_smc *smc, u64 cmd, u64 arg, KUNIT_EXPECT_TRUE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); ctx->calls++; + /* A written value as wide as it was written: WKTP's is one byte. */ if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) - ctx->seen[ctx->seen_count++] = cmd == SMC_MSG_WRITE_KEY ? - get_unaligned((u32 *)ctx->shmem) : arg; + ctx->seen[ctx->seen_count++] = cmd != SMC_MSG_WRITE_KEY ? arg : + get_unaligned((u32 *)ctx->shmem) & + GENMASK(8 * min_t(u64, size, sizeof(u32)) - 1, 0); if (ret_data && ctx->result >= 0) *ret_data = ctx->data; return ctx->result; @@ -222,12 +225,15 @@ static int macsmc_test_ap_power(struct apple_smc *smc, bool on) return ctx->ap_power_result; } -static void macsmc_sleep_order_test(struct kunit *test) +/* A sleep and its wake, for an SMC that takes WKTP and for one that does not. */ +static void macsmc_sleep_order(struct kunit *test, bool wktp) { struct macsmc_test_context *ctx = test->priv; + unsigned int n = 0; kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); ctx->result = 8; + ctx->smc.wktp = wktp; apple_smc_sleep_warn(&ctx->smc); KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), 0); @@ -236,15 +242,34 @@ static void macsmc_sleep_order_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); - /* As in the trace, values included. */ - KUNIT_ASSERT_EQ(test, ctx->seen_count, 7U); - KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); - KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(slpa)); - KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_IDLE); - KUNIT_EXPECT_EQ(test, ctx->seen[3], (u32)MACSMC_TEST_AP_ON); - KUNIT_EXPECT_EQ(test, ctx->seen[4], SMC_KEY(MBSE)); - KUNIT_EXPECT_EQ(test, ctx->seen[5], SMC_KEY(MBSW)); - KUNIT_EXPECT_EQ(test, ctx->seen[6], SMC_KEY(waka)); + /* + * Values included: WKTP's 2 comes before 'slpa', its 0 ends the wake. + * The lid is read at every wake, for the display. + */ + KUNIT_ASSERT_EQ(test, ctx->seen_count, wktp ? 10U : 8U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpw)); + if (wktp) + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 2U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(waka)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MSLD)); + if (wktp) + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 0U); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); +} + +static void macsmc_sleep_order_test(struct kunit *test) +{ + macsmc_sleep_order(test, false); +} + +static void macsmc_sleep_order_wktp_test(struct kunit *test) +{ + macsmc_sleep_order(test, true); } static void macsmc_test_outbox_poll(struct apple_smc *smc, const char *after, @@ -259,33 +284,274 @@ static void macsmc_sleep_warn_late_test(struct kunit *test) kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); kunit_activate_static_stub(test, apple_smc_outbox_poll, macsmc_test_outbox_poll); ctx->result = 8; + ctx->smc.wktp = true; - /* A sleep that cuts the debug link: slpw comes at the noirq suspend, before slpa. */ + /* Without last stops, slpw comes at the noirq suspend, right before the commit. */ ctx->smc.sleep_warn_late = true; KUNIT_EXPECT_EQ(test, apple_smc_sleep_enter(&ctx->smc), 0); KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warn_late); KUNIT_EXPECT_TRUE(test, ctx->smc.sleep_warned); - KUNIT_ASSERT_EQ(test, ctx->seen_count, 3U); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 4U); KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); - KUNIT_EXPECT_EQ(test, ctx->seen[1], SMC_KEY(slpa)); - KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[1], 2U); + KUNIT_EXPECT_EQ(test, ctx->seen[2], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[3], (u32)MACSMC_TEST_AP_IDLE); } -static void macsmc_sleep_refused_test(struct kunit *test) +static void macsmc_sleep_refused(struct kunit *test, bool wktp) { struct macsmc_test_context *ctx = test->priv; + unsigned int n = 0; kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); ctx->ap_power_result = -ETIME; ctx->result = 4; + ctx->smc.wktp = wktp; /* No answer: refuse, undo the notice and leave commands working. */ KUNIT_EXPECT_EQ(test, apple_smc_sleep_commit(&ctx->smc), -ETIME); KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); - KUNIT_ASSERT_EQ(test, ctx->seen_count, 3U); - KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpa)); - KUNIT_EXPECT_EQ(test, ctx->seen[1], (u32)MACSMC_TEST_AP_IDLE); - KUNIT_EXPECT_EQ(test, ctx->seen[2], (u32)MACSMC_TEST_AP_ON); + KUNIT_ASSERT_EQ(test, ctx->seen_count, wktp ? 4U : 3U); + if (wktp) + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 2U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_ON); + + /* The key the commit wrote is taken back when the sleep ends, once. */ + KUNIT_EXPECT_TRUE(test, ctx->smc.wake_silent); + apple_smc_wake_full(&ctx->smc); + apple_smc_wake_full(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + KUNIT_ASSERT_EQ(test, ctx->seen_count, wktp ? 5U : 3U); + if (wktp) + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 0U); +} + +static void macsmc_sleep_refused_test(struct kunit *test) +{ + macsmc_sleep_refused(test, false); +} + +static void macsmc_sleep_refused_wktp_test(struct kunit *test) +{ + macsmc_sleep_refused(test, true); +} + +static bool macsmc_test_no_commit(void) +{ + return false; +} + +static void macsmc_sleep_dry_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_sleep_commits, macsmc_test_no_commit); + ctx->result = 8; + ctx->smc.wktp = true; + + /* + * A sleep that ends short of the power loss, as a dry run does: the SMC + * gets neither the warning nor anything of the commit. + */ + ctx->smc.sleep_warn_late = true; + KUNIT_EXPECT_EQ(test, apple_smc_sleep_noirq(&ctx->smc), 0); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + apple_smc_wake_noirq(&ctx->smc); + apple_smc_sleep_end(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warn_late); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 0U); +} + +static bool macsmc_test_commit(void) +{ + return true; +} + +static void macsmc_sleep_refused_late_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + unsigned int n = 0; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_outbox_poll, macsmc_test_outbox_poll); + kunit_activate_static_stub(test, apple_smc_sleep_commits, macsmc_test_commit); + ctx->result = 8; + ctx->smc.wktp = true; + + /* + * A real sleep, committed in the noirq suspend, that something refuses after the + * commit: no power loss, and the SMC gets what a wake sends, in a wake's order and + * once. + */ + apple_smc_sleep_warn(&ctx->smc); + KUNIT_EXPECT_EQ(test, apple_smc_sleep_noirq(&ctx->smc), 0); + KUNIT_EXPECT_TRUE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_TRUE(test, ctx->smc.wake_silent); + apple_smc_wake_noirq(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + apple_smc_sleep_end(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 10U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpw)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 2U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_IDLE); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(waka)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MSLD)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 0U); +} + +/* The atomic writes of the sleep controller's last stop: no mutex, and only in atomic mode. */ +static int macsmc_test_write_atomic(struct apple_smc *smc, smc_key key, const void *buf, + size_t size) +{ + struct macsmc_test_context *ctx = container_of(smc, struct macsmc_test_context, smc); + + KUNIT_EXPECT_TRUE(kunit_get_current_test(), smc->atomic_mode); + KUNIT_EXPECT_FALSE(kunit_get_current_test(), mutex_is_locked(&smc->mutex)); + if (ctx->seen_count < ARRAY_SIZE(ctx->seen)) + ctx->seen[ctx->seen_count++] = size == sizeof(u8) ? *(const u8 *)buf : + get_unaligned((const u32 *)buf); + return ctx->atomic_result; +} + +static bool macsmc_test_at_stop(struct apple_smc *smc) +{ + return true; +} + +/* A sleep on a controller that runs last stops, up to where the noirq suspend has passed. */ +static void macsmc_last_stop_begin(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + kunit_activate_static_stub(test, apple_smc_ap_power, macsmc_test_ap_power); + kunit_activate_static_stub(test, apple_smc_outbox_poll, macsmc_test_outbox_poll); + kunit_activate_static_stub(test, apple_smc_sleep_commits, macsmc_test_commit); + kunit_activate_static_stub(test, apple_smc_commits_at_stop, macsmc_test_at_stop); + kunit_activate_static_stub(test, apple_smc_write_atomic, macsmc_test_write_atomic); + ctx->result = 8; + ctx->smc.wktp = true; + + /* The noirq suspend sends nothing and leaves the warning for the last stop. */ + ctx->smc.sleep_warn_late = true; + KUNIT_EXPECT_EQ(test, apple_smc_sleep_noirq(&ctx->smc), 0); + KUNIT_EXPECT_TRUE(test, ctx->smc.sleep_warn_late); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 0U); +} + +/* The wake after a last stop: a wake's commands in a wake's order, with the mutex again. */ +static void macsmc_last_stop_wake(struct kunit *test, unsigned int n) +{ + struct macsmc_test_context *ctx = test->priv; + + KUNIT_EXPECT_TRUE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_TRUE(test, ctx->smc.wake_silent); + KUNIT_EXPECT_FALSE(test, ctx->smc.atomic_mode); + KUNIT_EXPECT_FALSE(test, ctx->smc.atomic_pending); + apple_smc_wake_noirq(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + apple_smc_sleep_end(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + KUNIT_ASSERT_EQ(test, ctx->seen_count, n + 6); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_ON); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSE)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MBSW)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(waka)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(MSLD)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 0U); +} + +static void macsmc_last_stop_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + unsigned int n = 0; + + macsmc_last_stop_begin(test); + + /* The whole sequence comes from the last stop, in the noirq suspend's order, once. */ + KUNIT_EXPECT_EQ(test, apple_smc_last_stop(&ctx->smc), 0); + KUNIT_EXPECT_EQ(test, apple_smc_last_stop(&ctx->smc), 0); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warn_late); + KUNIT_EXPECT_TRUE(test, ctx->smc.sleep_warned); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 4U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpw)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], 2U); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], SMC_KEY(slpa)); + KUNIT_EXPECT_EQ(test, ctx->seen[n++], (u32)MACSMC_TEST_AP_IDLE); + + macsmc_last_stop_wake(test, n); +} + +static void macsmc_last_stop_refused_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + macsmc_last_stop_begin(test); + + /* + * Refused after the noirq suspend and before the last stops, as by a wake event or + * the enclave: the SMC was sent nothing, and the resume sends it nothing. + */ + apple_smc_wake_noirq(&ctx->smc); + apple_smc_sleep_end(&ctx->smc); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warn_late); + KUNIT_EXPECT_FALSE(test, ctx->smc.sleep_warned); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.wake_silent); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 0U); + + /* A last stop with nothing left for it sends nothing either. */ + KUNIT_EXPECT_EQ(test, apple_smc_last_stop(&ctx->smc), 0); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 0U); +} + +static void macsmc_last_stop_failed_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + macsmc_last_stop_begin(test); + + /* + * An SMC that answers nothing: a last stop cannot refuse, but each further step would + * wait out its timeout again with interrupts off. Only the warning is tried, nothing else + * is sent, no idle notice is owed to the wake, and the first failure is returned. + */ + ctx->atomic_result = -ETIMEDOUT; + ctx->smc.atomic_pending = true; + KUNIT_EXPECT_EQ(test, apple_smc_last_stop(&ctx->smc), -ETIMEDOUT); + KUNIT_ASSERT_EQ(test, ctx->seen_count, 1U); + KUNIT_EXPECT_EQ(test, ctx->seen[0], SMC_KEY(slpw)); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); + KUNIT_EXPECT_FALSE(test, ctx->smc.atomic_mode); + KUNIT_EXPECT_FALSE(test, ctx->smc.atomic_pending); + KUNIT_EXPECT_TRUE(test, ctx->smc.sleep_warned); +} + +/* The same when the answer to an earlier command is still missing (-EBUSY). */ +static void macsmc_last_stop_late_test(struct kunit *test) +{ + struct macsmc_test_context *ctx = test->priv; + + macsmc_last_stop_begin(test); + + ctx->atomic_result = -EBUSY; + KUNIT_EXPECT_EQ(test, apple_smc_last_stop(&ctx->smc), -EBUSY); + KUNIT_EXPECT_EQ(test, ctx->seen_count, 1U); + KUNIT_EXPECT_FALSE(test, ctx->smc.ap_notified); } static void macsmc_notice_test(struct kunit *test) @@ -396,8 +662,16 @@ static struct kunit_case macsmc_parser_cases[] = { KUNIT_CASE(macsmc_sram_value_test), KUNIT_CASE(macsmc_key_info_test), KUNIT_CASE(macsmc_sleep_order_test), + KUNIT_CASE(macsmc_sleep_order_wktp_test), KUNIT_CASE(macsmc_sleep_warn_late_test), KUNIT_CASE(macsmc_sleep_refused_test), + KUNIT_CASE(macsmc_sleep_refused_wktp_test), + KUNIT_CASE(macsmc_sleep_dry_test), + KUNIT_CASE(macsmc_sleep_refused_late_test), + KUNIT_CASE(macsmc_last_stop_test), + KUNIT_CASE(macsmc_last_stop_refused_test), + KUNIT_CASE(macsmc_last_stop_failed_test), + KUNIT_CASE(macsmc_last_stop_late_test), KUNIT_CASE(macsmc_notice_test), KUNIT_CASE(macsmc_lid_wake_test), KUNIT_CASE(macsmc_wake_reason_test), diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 04e90a97cc313..c319d92ac8e1a 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +91,26 @@ static const struct mfd_cell apple_smc_t6050_described_devs[] = { MFD_CELL_OF("macsmc-rtc", NULL, NULL, 0, 0, "apple,smc-rtc"), }; +/* What this driver has to know of one SoC's SMC that the others do not share. */ +struct apple_smc_hw { + /* + * The SMC takes the WKTP key at the sleep request and once a wake is a + * full one: see apple_smc_sleep_commit() and apple_smc_wake_full(). + */ + bool wktp; + /* + * This SoC's sleep controller stops its coprocessors at the commit, behind + * every refusal, and the SMC's sleep commit goes with them: see + * apple_smc_last_stop(). + */ + bool commit_at_stop; +}; + +static const struct apple_smc_hw apple_smc_t6041_hw = { + .wktp = true, + .commit_at_stop = true, +}; + static int apple_smc_add_described_devices(struct apple_smc *smc, const struct mfd_cell *cells, size_t count) { @@ -356,11 +378,13 @@ EXPORT_SYMBOL(apple_smc_enter_atomic); int apple_smc_write_atomic(struct apple_smc *smc, smc_key key, const void *buf, size_t size) { + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_write_atomic, smc, key, buf, size); + guard(spinlock_irqsave)(&smc->lock); u8 result; int ret, poll_ret; - u64 msg; - ktime_t t0; + u64 msg, t0; s64 us; if (size > SMC_MAX_SIZE || size == 0) @@ -404,12 +428,13 @@ int apple_smc_write_atomic(struct apple_smc *smc, smc_key key, const void *buf, return ret; } - t0 = ktime_get(); + /* A sleep's last stop writes with timekeeping suspended, which ktime_get() warns of. */ + t0 = ktime_get_mono_fast_ns(); ret = read_poll_timeout_atomic(apple_rtkit_poll, poll_ret, poll_ret < 0 || !smc->atomic_pending, 100, SMC_ATOMIC_TIMEOUT_MS * USEC_PER_MSEC, false, smc->rtk); - us = ktime_us_delta(ktime_get(), t0); + us = div_u64(ktime_get_mono_fast_ns() - t0, NSEC_PER_USEC); if (us > 100 * USEC_PER_MSEC || ret < 0 || smc->atomic_pending) dev_warn(smc->dev, "atomic: key %p4ch reply after %lld us (ret %d)\n", &key, us, ret); @@ -572,11 +597,78 @@ static const struct apple_smc_wake_reason apple_smc_t8132_wake_reasons[] = { { 44, "USB2_wake" }, { 45, "aupo" }, { 48, "rtc" }, { } }; +/* + * The kernel that loads a hibernation image binds this driver like any boot + * and so holds a live RTKit session with the SMC when it jumps into the image. + * The restored kernel then asks the firmware to start again (IOP power INIT) + * and waits for a HELLO that a firmware already in a session never sends. So + * the loading kernel hands the SMC over quiesced, the state an unbind leaves + * it in (apple_smc_rtkit_shutdown()) and from which a wake starts a new + * session. + */ +static bool apple_smc_handed_over; + +/* + * The handshake with an SMC that was just woken, at probe and after a resume + * from disk alike: its endpoint, the initialization and the key count. The + * firmware is a new instance each time, and nothing says that its key table is + * the one of the instance before. + */ +static int apple_smc_handshake(struct apple_smc *smc) +{ + struct device *dev = smc->dev; + __be32 count = 0; + int ret; + + ret = apple_rtkit_start_ep(smc->rtk, SMC_ENDPOINT); + if (ret) { + dev_err(dev, "Failed to start SMC endpoint: %d\n", ret); + return ret; + } + + ret = apple_rtkit_send_message(smc->rtk, SMC_ENDPOINT, + FIELD_PREP(SMC_MSG, SMC_MSG_INITIALIZE), NULL, false); + if (ret) { + dev_err(dev, "Failed to send init message: %d\n", ret); + return ret; + } + + if (wait_for_completion_timeout(&smc->init_done, msecs_to_jiffies(SMC_TIMEOUT_MS)) == 0) { + dev_err(dev, "Timed out initializing SMC\n"); + return -ETIMEDOUT; + } + + if (smc->boot_stage != APPLE_SMC_INITIALIZED) { + dev_err(dev, "SMC failed to boot successfully, boot stage=%d\n", smc->boot_stage); + return -EIO; + } + + ret = apple_smc_read(smc, SMC_KEY(#KEY), &count, sizeof(count)); + if (ret >= 0 && ret != sizeof(count)) + ret = -EIO; + if (ret < 0) { + dev_err(dev, "Failed to get key count: %d\n", ret); + return ret; + } + smc->key_count = be32_to_cpu(count); + + return 0; +} + +static int apple_smc_last_stop(void *data); + +static void apple_smc_last_stop_unregister(void *data) +{ + struct apple_smc *smc = data; + + apple_sleep_coproc_unregister(smc->dev); +} + static int apple_smc_probe(struct platform_device *pdev) { + const struct apple_smc_hw *hw = device_get_match_data(&pdev->dev); struct device *dev = &pdev->dev; struct apple_smc *smc; - __be32 count = 0; bool t6050; int ret; @@ -585,6 +677,7 @@ static int apple_smc_probe(struct platform_device *pdev) return -ENOMEM; smc->wake_reasons = of_device_is_compatible(dev->of_node, "apple,t8132-smc") ? apple_smc_t8132_wake_reasons : apple_smc_t8140_wake_reasons; + smc->wktp = hw && hw->wktp; mutex_init(&smc->mutex); spin_lock_init(&smc->lock); @@ -606,39 +699,14 @@ static int apple_smc_probe(struct platform_device *pdev) if (ret) return ret; - ret = apple_rtkit_start_ep(smc->rtk, SMC_ENDPOINT); - if (ret) - return dev_err_probe(dev, ret, "Failed to start SMC endpoint"); - init_completion(&smc->init_done); init_completion(&smc->cmd_done); - - ret = apple_rtkit_send_message(smc->rtk, SMC_ENDPOINT, - FIELD_PREP(SMC_MSG, SMC_MSG_INITIALIZE), NULL, false); - if (ret) - return dev_err_probe(dev, ret, "Failed to send init message"); - - if (wait_for_completion_timeout(&smc->init_done, msecs_to_jiffies(SMC_TIMEOUT_MS)) == 0) { - dev_err(dev, "Timed out initializing SMC"); - return -ETIMEDOUT; - } - - if (smc->boot_stage != APPLE_SMC_INITIALIZED) { - dev_err(dev, "SMC failed to boot successfully, boot stage=%d\n", smc->boot_stage); - return -EIO; - } - dev_set_drvdata(&pdev->dev, smc); BLOCKING_INIT_NOTIFIER_HEAD(&smc->event_handlers); - ret = apple_smc_read(smc, SMC_KEY(#KEY), &count, sizeof(count)); - if (ret >= 0 && ret != sizeof(count)) { - count = 0; - ret = -EIO; - } - if (ret < 0) - return dev_err_probe(smc->dev, ret, "Failed to get key count"); - smc->key_count = be32_to_cpu(count); + ret = apple_smc_handshake(smc); + if (ret) + return ret; /* Notifications, GPIO and power management are not qualified on T8152. */ if (of_device_is_compatible(dev->of_node, "apple,t8152-smc")) @@ -651,6 +719,19 @@ static int apple_smc_probe(struct platform_device *pdev) if (ret) return ret; + /* + * Without the registration (no sleep controller with last stops for this kernel, + * or no room in its table) the noirq suspend commits, as on every other SoC. + */ + if (hw && hw->commit_at_stop && + !apple_sleep_coproc_register_first(dev, APPLE_SLEEP_FIRST_SMC, apple_smc_last_stop, + smc)) { + smc->commit_at_stop = true; + ret = devm_add_action_or_reset(dev, apple_smc_last_stop_unregister, smc); + if (ret) + return ret; + } + /* Enable notifications */ apple_smc_write_flag(smc, SMC_KEY(NTAP), true); ret = devm_add_action_or_reset(dev, apple_smc_disable_notifications, smc); @@ -678,9 +759,101 @@ static int apple_smc_probe(struct platform_device *pdev) ARRAY_SIZE(apple_smc_t6050_described_devs)); } +static int apple_smc_restore(struct device *dev); + +static int apple_smc_freeze(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + + /* Only the image-loading kernel, right before it enters the image. */ + if (pm_hibernate_is_loading() && apple_rtkit_is_running(smc->rtk)) { + int ret; + + apple_smc_write_flag(smc, SMC_KEY(NTAP), false); + ret = apple_rtkit_quiesce(smc->rtk); + if (ret) { + /* + * Failing here would abandon the image, whose signature is + * already gone, and leave this kernel without a session: enter + * the image anyway, its restore tries a new session. + */ + dev_err(dev, "could not quiesce the SMC for the image: %d\n", ret); + return 0; + } + apple_smc_handed_over = true; + dev_info(dev, "SMC quiesced for the restored kernel\n"); + return 0; + } + + /* + * Hibernating: the SMC keeps running, and with it thermal and power + * management, while the image is written and until the power-off + * (.poweroff shares this). Nothing is saved: the boot that loads the + * image restarts the SMC, and apple_smc_restore() starts a new session. + */ + return 0; +} + +static int apple_smc_thaw(struct device *dev) +{ + /* The image could not be entered after the hand-over: start a new session. */ + if (apple_smc_handed_over) { + apple_smc_handed_over = false; + return apple_smc_restore(dev); + } + + /* The image was written or abandoned: .freeze left the session running. */ + return 0; +} + +static int apple_smc_restore(struct device *dev) +{ + struct apple_smc *smc = dev_get_drvdata(dev); + int ret; + + /* + * The boot that loaded the image restarted the SMC. The RTKit session + * in the image belongs to a firmware instance that no longer runs, and + * its power state still reads running, which apple_rtkit_wake() + * refuses. Reset the transport to the state probe starts from (power + * state off, no endpoints, mailbox receive restarted) and repeat + * probe's handshake with the new instance. + */ + ret = apple_rtkit_reinit(smc->rtk); + if (ret) { + dev_err(dev, "Failed to reinit RTKit on restore: %d\n", ret); + return ret; + } + + smc->boot_stage = APPLE_SMC_BOOTING; + + ret = apple_rtkit_wake(smc->rtk); + if (ret) { + dev_err(dev, "Failed to wake up SMC on restore: %d\n", ret); + return ret; + } + + reinit_completion(&smc->init_done); + reinit_completion(&smc->cmd_done); + + ret = apple_smc_handshake(smc); + if (ret) + return ret; + + /* + * Re-enable the firmware-side notification flag. The event_handlers + * notifier chain and the sub-device registrations on it survive in the + * restored image, so only the firmware flag has to be re-armed. + */ + apple_smc_write_flag(smc, SMC_KEY(NTAP), true); + + return 0; +} + static const struct of_device_id apple_smc_of_match[] = { { .compatible = "apple,t8152-smc" }, { .compatible = "apple,t6050-smc" }, + { .compatible = "apple,t6041-smc", .data = &apple_smc_t6041_hw }, { .compatible = "apple,t8103-smc" }, { .compatible = "apple,smc" }, {}, @@ -690,8 +863,8 @@ MODULE_DEVICE_TABLE(of, apple_smc_of_match); /* * The SMC keeps running through a firmware-driven suspend. It is told first and * stops last, and on a wake it hears first. In order, measured on T8140 and - * unverified on other SoCs: MBSE = slpw before any other coprocessor is told; - * MBSE = slpa and the AP power notice after all the others; on a wake the notice, + * unverified on other SoCs: MBSE = slpw, MBSE = slpa and the AP power notice after + * all the other coprocessors (slpw goes ahead of them); on a wake the notice, * MBSE and MBSW read, MBSE = waka. The noirq phases give that order whatever the * device order is, so these commands poll. What the values mean is inferred * from their letters. @@ -702,15 +875,22 @@ MODULE_DEVICE_TABLE(of, apple_smc_of_match); * The debug link's cut, on a debug cable only, makes the SMC see a cable pulled and plugged * back. With the cut after slpw, the SMC woke the AP right after the commit, for USB-C_plug * or acdetach, in every such sleep (J700, 223 of 223), even with its outbox quiet for 500 ms - * before the notice. A sleep that cuts the link warns the SMC only after the cut. + * before the notice. So the warning comes after the cut. + * + * A T6041 SMC takes neither the warning nor the commit back: see apple_smc_sleep_enter() and + * apple_smc_sleep_commits(). After the noirq suspend a sleep can still be refused: by a wake + * event, by a CPU that does not park, by the sleep controller's checks at its commit, by the + * enclave before its sleep request. So where the controller stops coprocessors at its commit, + * with interrupts off, one CPU up and every refusal behind it, the whole sequence is sent + * from there, ahead of the other coprocessors' stops: apple_smc_last_stop(). */ /* The gaps macOS leaves before the AP idle notice and after it. */ #define SMC_SLEEP_ARM_MS 6 #define SMC_SLEEP_IDLE_MS 30 /* - * Between a late warning (a sleep that cut the debug link) and slpa. macOS's shortest gap - * was 770 ms; the warning at .prepare leaves 150-500 ms, and those sleeps run to the alarm. + * Between the warning and slpa. Sleeps with 150-500 ms between them ran to the alarm; + * 770 ms is known to work. */ #define SMC_SLEEP_WARN_MS 200 @@ -720,9 +900,15 @@ MODULE_DEVICE_TABLE(of, apple_smc_of_match); static int apple_smc_ap_power(struct apple_smc *smc, bool on) { + unsigned int us; + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) KUNIT_STATIC_STUB_REDIRECT(apple_smc_ap_power, smc, on); + /* With interrupts off there is only the idle notice: see apple_smc_last_stop(). */ + if (smc->atomic_mode) + return on ? -EINVAL : apple_rtkit_ap_idle_polled(smc->rtk, &us); + return smc->polled ? apple_rtkit_ap_power_noirq(smc->rtk, on) : apple_rtkit_ap_power(smc->rtk, on); } @@ -750,6 +936,36 @@ static void apple_smc_write_mbse(struct apple_smc *smc, smc_key value) dev_err(smc->dev, "Failed to issue MBSE = %p4ch: %d\n", &value, ret); } +/* + * SMC key pmVC, the request for the VDD_CIO rail: the enable mask | 0x100 + * (0x101 on the J616s), written while an ATC port's always-on side is powered, also at sleep. Linux + * never asks for the rail, which is power-gated otherwise (ADT vdd-cio-rail-pg), and a rail + * the SMC drops during the sleep transition races the sleep engine's hand-over of the ATC + * ports. + */ +#define SMC_PMVC_REQUEST 0x101 + +/** + * apple_smc_request_pmvc() - ask the SMC to keep the VDD_CIO rail up + * @smc: the SMC + * + * Writes pmVC at the moment the caller picks; it does not hook the SMC's own suspend + * callbacks, so a sleep controller that has to issue it ahead of the device suspend + * phase keeps that order. + * + * Return: 0 on success, a negative errno otherwise. + */ +int apple_smc_request_pmvc(struct apple_smc *smc) +{ + int ret; + + ret = apple_smc_write_u32(smc, __SMC_KEY('p', 'm', 'V', 'C'), SMC_PMVC_REQUEST); + if (ret < 0) + dev_warn(smc->dev, "pmVC: write %#x failed: %d\n", SMC_PMVC_REQUEST, ret); + return ret; +} +EXPORT_SYMBOL(apple_smc_request_pmvc); + static void apple_smc_sleep_warn(struct apple_smc *smc) { apple_smc_write_mbse(smc, SMC_KEY(slpw)); @@ -780,10 +996,26 @@ static int apple_smc_notice_idle(struct apple_smc *smc) return ret; } +/* + * WKTP, one byte, the value for power state 0. It goes first at the sleep request, + * ahead of the other sleep keys ('slpa' among them) and the AP idle notice. Only for + * an SMC known to take it: T6041's. + */ +#define SMC_WKTP_SLEEP 2 +/* And once a wake is a full one: see apple_smc_wake_full(). */ +#define SMC_WKTP_AWAKE 0 + static int apple_smc_sleep_commit(struct apple_smc *smc) { int ret; + /* Whatever is written here, the SMC runs the wake after this silent until it is told. */ + smc->wake_silent = true; + if (smc->wktp) { + ret = apple_smc_write_u8(smc, SMC_KEY(WKTP), SMC_WKTP_SLEEP); + if (ret) + dev_err(smc->dev, "Failed to write WKTP = %u: %d\n", SMC_WKTP_SLEEP, ret); + } apple_smc_write_mbse(smc, SMC_KEY(slpa)); /* Device interrupts are off here. */ mdelay(SMC_SLEEP_ARM_MS); @@ -795,6 +1027,26 @@ static int apple_smc_sleep_commit(struct apple_smc *smc) return ret; } +/* + * WKTP once a wake is a full one: 0, where a wake that stays dark keeps the key nonzero, + * Without the 0 the SMC treats all the time awake as a power nap, whether + * the sleep wrote its 2 or not: it keeps its own fan policy against a manual fan request + * (the mode request times out), and under thermal load it requests a sleep once a second. + */ +static void apple_smc_wake_full(struct apple_smc *smc) +{ + int ret; + + if (!smc->wake_silent) + return; + smc->wake_silent = false; + if (!smc->wktp) + return; + ret = apple_smc_write_u8(smc, SMC_KEY(WKTP), SMC_WKTP_AWAKE); + if (ret) + dev_err(smc->dev, "Failed to write WKTP = %u: %d\n", SMC_WKTP_AWAKE, ret); +} + static int apple_smc_wake_reason_names(const struct apple_smc *smc, u64 record, char *buf, size_t size) { @@ -902,6 +1154,9 @@ static void apple_smc_sleep_wake(struct apple_smc *smc) apple_smc_lid_wake(smc); if (apple_smc_port_reasons_only(smc, smc->wake_record)) apple_sleep_debug_link_cut_wake(); + /* A dark wake sleeps again, and its commit writes the key anyway. */ + if (!pm_dark_wake()) + apple_smc_wake_full(smc); /* A dark wake isn't one: with wakeup_count in use, it would abort the sleep after it. */ if (smc->wake_record && !pm_dark_wake()) pm_wakeup_event(smc->dev, 0); @@ -913,29 +1168,35 @@ static int apple_smc_prepare(struct device *dev) if (!pm_suspend_via_firmware()) return 0; - if (apple_sleep_debug_link_up()) - smc->sleep_warn_late = true; - else - apple_smc_sleep_warn(smc); + /* Warned with the commit, not here: see apple_smc_sleep_enter(). */ + smc->sleep_warn_late = true; return 0; } /* Warned, and a refusal came before the notice: tell it the sleep is off. */ -static void apple_smc_complete(struct device *dev) +static void apple_smc_sleep_end(struct apple_smc *smc) { - struct apple_smc *smc = dev_get_drvdata(dev); - + if (smc->commit_at_stop) + apple_sleep_coproc_set_ready(smc->dev, false); smc->sleep_warn_late = false; if (smc->sleep_warned) { apple_smc_write_mbse(smc, SMC_KEY(waka)); smc->sleep_warned = false; } + /* A commit the SMC refused, or dark wakes that ended in a refusal. */ + apple_smc_wake_full(smc); +} + +static void apple_smc_complete(struct device *dev) +{ + apple_smc_sleep_end(dev_get_drvdata(dev)); } /* * For at least @min_ms, then until the outbox is quiet for @quiet_ms. The events go to their - * handlers as usual, through the RTKit worker. + * handlers as usual, through the RTKit worker. With interrupts off, in atomic mode, that + * worker runs them once the sleep is over. */ static void apple_smc_outbox_poll(struct apple_smc *smc, const char *after, unsigned int min_ms, unsigned int quiet_ms) @@ -955,18 +1216,147 @@ static void apple_smc_outbox_poll(struct apple_smc *smc, const char *after, } else { quiet++; } - usleep_range(1000, 1200); + if (smc->atomic_mode) + mdelay(1); + else + usleep_range(1000, 1200); } /* A debug cable only. */ dev_info(smc->dev, "%s: %u messages in %u ms, the last at %u ms%s\n", after, count, ms, last, ms >= SMC_OUTBOX_SETTLE_MS ? ", still busy" : ""); } -/* The noirq suspend's part, polled: the cut, a late warning, the commit's notices. */ +/* + * Whether the sleep controller's last stops send this sleep's commit: the SMC is registered + * for one, and the controller runs them for this sleep. + */ +static bool apple_smc_commits_at_stop(struct apple_smc *smc) +{ + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_commits_at_stop, smc); + + return smc->commit_at_stop && apple_sleep_commit_stops_coprocs(); +} + +static int apple_smc_stop_write(struct apple_smc *smc, smc_key key, const void *buf, size_t size) +{ + int ret = apple_smc_write_atomic(smc, key, buf, size); + + if (ret < 0) + dev_err(smc->dev, "Failed to write %p4ch at the last stop: %d\n", &key, ret); + + return ret < 0 ? ret : 0; +} + +/* + * The warning and the commit as one of the sleep controller's last stops: interrupts are + * off, one CPU is up, the enclave has its sleep request and nothing can refuse any more. So + * an SMC that got any of this also gets the power loss that brings it out of its sleep + * state. The order and the gaps are the noirq suspend's: apple_smc_sleep_enter() and + * apple_smc_sleep_commit(). + * + * Nothing sleeps here and the RTKit worker does not run, so the writes are the atomic ones + * and the idle notice's answer is polled for. Every other user of the SMC is suspended, and + * the mutex is not taken. An event the SMC sends meanwhile is taken from its outbox and + * reaches its handler after the wake. + * + * A last stop cannot refuse, and the sleep goes on whatever the SMC answers. So does this: + * a step that fails is logged and the rest is sent all the same, since an SMC that took + * the warning leaves its sleep state only through the rest of the sequence and the power + * loss. The one exception is an SMC that does not answer at all: a write waits up to + * SMC_ATOMIC_TIMEOUT_MS for its answer, so after the first timeout the rest is left out + * instead of waiting that long again for each step. The wake sends its part, as after any + * sleep, once this has started. Returns the first failure. + */ +static bool apple_smc_stop_dead(int ret) +{ + /* No answer within SMC_ATOMIC_TIMEOUT_MS, or the one before is still missing. */ + return ret == -ETIMEDOUT || ret == -EBUSY; +} + +static int apple_smc_last_stop(void *data) +{ + const u32 slpw = SMC_KEY(slpw), slpa = SMC_KEY(slpa); + const u8 wktp = SMC_WKTP_SLEEP; + struct apple_smc *smc = data; + int ret, err; + + /* Not left for here: the noirq suspend sent it, or this suspend is not a sleep. */ + if (!smc->sleep_warn_late) + return 0; + smc->sleep_warn_late = false; + + scoped_guard(spinlock_irqsave, &smc->lock) + smc->atomic_mode = true; + + err = apple_smc_stop_write(smc, SMC_KEY(MBSE), &slpw, sizeof(slpw)); + smc->sleep_warned = true; + if (apple_smc_stop_dead(err)) + goto dead; + apple_smc_outbox_poll(smc, "sleep warning at the last stop", SMC_SLEEP_WARN_MS, 0); + + /* As in apple_smc_sleep_commit(). */ + smc->wake_silent = true; + if (smc->wktp) { + ret = apple_smc_stop_write(smc, SMC_KEY(WKTP), &wktp, sizeof(wktp)); + err = err ?: ret; + if (apple_smc_stop_dead(ret)) + goto dead; + } + ret = apple_smc_stop_write(smc, SMC_KEY(MBSE), &slpa, sizeof(slpa)); + err = err ?: ret; + if (apple_smc_stop_dead(ret)) + goto dead; + mdelay(SMC_SLEEP_ARM_MS); + + /* Not taken back if the notice fails: the sleep goes on, and its wake sends the other. */ + smc->ap_notified = true; + ret = apple_smc_ap_power(smc, false); + if (ret) + dev_err(smc->dev, "SMC did not take the sleep notice at the last stop: %d\n", ret); + else + mdelay(SMC_SLEEP_IDLE_MS); + err = err ?: ret; + goto out; + +dead: + /* + * An SMC that answers nothing for SMC_ATOMIC_TIMEOUT_MS takes nothing more: each further + * write would wait that long again, and again for the answer still missing, all with + * interrupts off. The sleep goes on without the rest. + */ + dev_err(smc->dev, "SMC does not answer at the last stop, the rest is left out\n"); + err = err ?: -ETIMEDOUT; +out: + /* + * The wake's commands are the usual ones. An answer that never came is given up: if + * it comes after all, it fails the sequence check of the command it lands on. + */ + scoped_guard(spinlock_irqsave, &smc->lock) { + smc->atomic_pending = false; + smc->atomic_mode = false; + } + + return err; +} + +/* + * The noirq suspend's part, polled: the cut, the warning, the commit's notices. Where the + * sleep controller's last stops send the warning and the commit, only the cut is left here: + * it sleeps, and it has to come before the warning. + * + * The warning is not sent earlier, although the SMC could take it then. A T6041 SMC that + * was warned and then told that the sleep was off, with no commit between, stopped running + * the fans (0 rpm at a die temperature of 110 C) and refused a fan request, until a later + * sleep had completed: nothing sent to it takes the warning back. So it goes where the + * commit goes, after every device has had its chance to refuse. + */ static int apple_smc_sleep_enter(struct apple_smc *smc) { if (apple_sleep_debug_link_cut()) apple_smc_outbox_poll(smc, "debug link cut", 0, SMC_OUTBOX_QUIET_MS); + if (apple_smc_commits_at_stop(smc)) + return 0; if (smc->sleep_warn_late) { smc->sleep_warn_late = false; apple_smc_sleep_warn(smc); @@ -976,13 +1366,37 @@ static int apple_smc_sleep_enter(struct apple_smc *smc) return apple_smc_sleep_commit(smc); } -static int apple_smc_suspend_noirq(struct device *dev) +/* + * Whether this suspend goes on to the SoC's power loss. Only then is the SMC given its sleep + * commit, which nothing sent to it takes back: a T6041 SMC that took the commit and then saw + * no power loss kept every sensor report at its last value until a later sleep had completed, + * although it got the notices of a wake as after any sleep. So the commit is among the stops + * that only a power cycle undoes, and a dry run, which ends short of the power loss by + * design, leaves it out. + */ +static bool apple_smc_sleep_commits(void) +{ + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_sleep_commits); + + return apple_sleep_power_loss(); +} + +static int apple_smc_sleep_noirq(struct apple_smc *smc) { - struct apple_smc *smc = dev_get_drvdata(dev); int ret; if (!smc->sleep_warned && !smc->sleep_warn_late) return 0; + /* + * Nothing of the SMC's is in the way of the sleep controller's stops, in a dry run's + * checks as in a sleep. Its last stop runs only where a sleep is committed. + */ + if (smc->commit_at_stop) + apple_sleep_coproc_set_ready(smc->dev, true); + /* A sleep that stops short of the commit sends the SMC nothing. */ + if (!apple_smc_sleep_commits()) + return 0; apple_smc_set_polled(smc, true); ret = apple_smc_sleep_enter(smc); @@ -991,22 +1405,49 @@ static int apple_smc_suspend_noirq(struct device *dev) return ret; } -static int apple_smc_resume_noirq(struct device *dev) +static int apple_smc_suspend_noirq(struct device *dev) { - struct apple_smc *smc = dev_get_drvdata(dev); + return apple_smc_sleep_noirq(dev_get_drvdata(dev)); +} - if (smc->ap_notified) { - apple_smc_set_polled(smc, true); - apple_smc_sleep_wake(smc); - apple_smc_set_polled(smc, false); - } +static void apple_smc_wake_noirq(struct apple_smc *smc) +{ + if (!smc->ap_notified) + return; + + /* + * Committed, and no power loss followed: a check that comes after this device's + * suspend refused the sleep, or a wake event got in first. The SMC gets the notices + * of a wake all the same, which is all there is to send, but it leaves its sleep + * state only through a power loss: see apple_smc_sleep_commits(). A commit sent from + * the sleep controller's last stop has every refusal behind it. + */ + if (!pm_resume_via_firmware()) + dev_warn(smc->dev, "sleep refused after the SMC's commit: its sensor reports may stand still until a sleep completes\n"); + + apple_smc_set_polled(smc, true); + apple_smc_sleep_wake(smc); + apple_smc_set_polled(smc, false); +} + +static int apple_smc_resume_noirq(struct device *dev) +{ + apple_smc_wake_noirq(dev_get_drvdata(dev)); return 0; } +/* + * Suspend-to-RAM commits in the noirq phase or from the sleep controller's last stop; + * hibernation restarts the session in the normal one. + */ static const struct dev_pm_ops apple_smc_pm_ops = { .prepare = pm_sleep_ptr(apple_smc_prepare), .complete = pm_sleep_ptr(apple_smc_complete), + .freeze = pm_sleep_ptr(apple_smc_freeze), + .thaw = pm_sleep_ptr(apple_smc_thaw), + .poweroff = pm_sleep_ptr(apple_smc_freeze), + .restore = pm_sleep_ptr(apple_smc_restore), NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_smc_suspend_noirq, apple_smc_resume_noirq) }; diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index 3e8ab5a9b933b..3b686a0218940 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -54,8 +54,21 @@ struct apple_gpio_pinctrl { /* The pins' words across a firmware-driven suspend */ u32 *saved; unsigned long *saved_valid; + /* The regmap caches the block: it holds what this driver set, not the hardware. */ + bool cached; + /* A pin is written back in steps: see apple_gpio_safe_reconfigure(). */ + bool staged; /* Pins this driver has written. Others belong to someone else. */ unsigned long *touched; + /* + * A block whose every pin has to be put back after a sleep that lost its + * power (apple,resume-restore-all), but those in gpio-reserved-ranges. + */ + bool restore_all; + unsigned long *no_restore; + /* The block's word at REG_GLOBAL, kept across a sleep as well. */ + u32 saved_global; + bool saved_global_valid; struct apple_gpio_irq_group irqgrps[]; }; @@ -82,6 +95,12 @@ struct apple_gpio_pinctrl { #define REG_GPIOx_LOCK BIT(21) #define REG_GPIOx_DRIVE_STRENGTH1 GENMASK(23, 22) #define REG_IRQ(g, x) (0x800 + 0x40 * (g) + 4 * ((x) >> 5)) +/* + * One word past the pins and the interrupt groups. The block loses it with its + * power, and it has to be back before the pins are: with it lost, no pin reads + * its input any more. It is 1 on a running J616s. + */ +#define REG_GLOBAL 0xc48 static const struct regmap_config regmap_config = { .reg_bits = 32, @@ -403,6 +422,11 @@ struct apple_gpio_hw { * only within the valid GPIO registers minus gpio-reserved-ranges. */ bool no_regcache; + /* + * A pin written back after a sleep or a resume from disk takes its word in + * steps (apple_gpio_safe_reconfigure()). Elsewhere the word goes back whole. + */ + bool staged_restore; }; /* T8140 (A18 Pro) and T8132 (M4) both have faulting slots in this block. */ @@ -410,6 +434,11 @@ static const struct apple_gpio_hw apple_gpio_uncached_hw = { .no_regcache = true, }; +/* T6040 and T6041 (M4 Pro and Max) */ +static const struct apple_gpio_hw apple_gpio_t6041_hw = { + .staged_restore = true, +}; + static int apple_gpio_regmap_access(struct device *dev, unsigned int npins, struct regmap_config *config) { @@ -603,13 +632,36 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(pctl->map), "Failed to create regmap\n"); - /* Only with the regmap reaching the hardware: a cache would hide the loss. */ - if (hw && hw->no_regcache) { - pctl->saved = devm_kcalloc(&pdev->dev, npins, sizeof(*pctl->saved), GFP_KERNEL); - pctl->saved_valid = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); - pctl->touched = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); - if (!pctl->saved || !pctl->saved_valid || !pctl->touched) + /* + * A cache would hide the loss: on a cached block the resume compares and + * writes the hardware past it. + */ + pctl->cached = !(hw && hw->no_regcache); + pctl->staged = hw && hw->staged_restore; + pctl->saved = devm_kcalloc(&pdev->dev, npins, sizeof(*pctl->saved), GFP_KERNEL); + pctl->saved_valid = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + if (!pctl->saved || !pctl->saved_valid) + return -ENOMEM; + /* A resume from disk also puts pins back, from the cache where there is one. */ + pctl->touched = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + if (!pctl->touched) + return -ENOMEM; + pctl->restore_all = of_property_read_bool(pdev->dev.of_node, "apple,resume-restore-all"); + if (pctl->restore_all) { + int n = of_property_count_u32_elems(pdev->dev.of_node, "gpio-reserved-ranges"); + u32 start, count; + + pctl->no_restore = devm_bitmap_zalloc(&pdev->dev, npins, GFP_KERNEL); + if (!pctl->no_restore) return -ENOMEM; + /* The pins the firmware lists as no-restore, as (first pin, count) pairs. */ + for (i = 0; n > 0 && i + 1 < (unsigned int)n; i += 2) + if (!of_property_read_u32_index(pdev->dev.of_node, "gpio-reserved-ranges", + i, &start) && + !of_property_read_u32_index(pdev->dev.of_node, "gpio-reserved-ranges", + i + 1, &count) && + start < npins && count <= npins - start) + bitmap_set(pctl->no_restore, start, count); } for (i = 0; i < npins; i++) { @@ -657,12 +709,44 @@ static const struct of_device_id apple_gpio_pinctrl_of_match[] = { { .compatible = "apple,t8140-pinctrl", .data = &apple_gpio_uncached_hw }, { .compatible = "apple,t8132-pinctrl", .data = &apple_gpio_uncached_hw }, { .compatible = "apple,t6050-pinctrl", .data = &apple_gpio_uncached_hw }, + { .compatible = "apple,t6041-pinctrl", .data = &apple_gpio_t6041_hw }, { .compatible = "apple,t8103-pinctrl", }, { .compatible = "apple,pinctrl", }, { } }; MODULE_DEVICE_TABLE(of, apple_gpio_pinctrl_of_match); +/* + * A pin's whole word, in the order the pin takes it after a power loss: the + * configuration first with the input buffer as it is (off after a power loss), then the + * input enable in a write of its own, the lock last. A locked pin is left alone. Written + * in one go, with the input enable among the rest, the pin reads 0 for good after the wake. + * That is established for the T6041 blocks; any other block gets its word whole, as before. + */ +static void apple_gpio_safe_reconfigure(struct apple_gpio_pinctrl *pctl, unsigned int pin, + u32 old, u32 cfg) +{ + u32 val = cfg & ~(REG_GPIOx_LOCK | REG_GPIOx_INPUT_ENABLE); + + if (!pctl->staged) { + regmap_write(pctl->map, REG_GPIO(pin), cfg); + return; + } + if (old & REG_GPIOx_LOCK) + return; + if (!(cfg & REG_GPIOx_INPUT_ENABLE)) { + regmap_write(pctl->map, REG_GPIO(pin), + old & ~(REG_GPIOx_LOCK | REG_GPIOx_INPUT_ENABLE)); + regmap_write(pctl->map, REG_GPIO(pin), val); + } else { + regmap_write(pctl->map, REG_GPIO(pin), (old & REG_GPIOx_INPUT_ENABLE) | val); + if (!(old & REG_GPIOx_INPUT_ENABLE)) + regmap_write(pctl->map, REG_GPIO(pin), cfg & ~REG_GPIOx_LOCK); + } + if (cfg & REG_GPIOx_LOCK) + regmap_write(pctl->map, REG_GPIO(pin), cfg); +} + /* * A firmware-driven suspend powers this block off, and neither the pin core nor * a consumer sets a pin up again. Slots that fault aren't readable in the regmap, @@ -673,15 +757,34 @@ static int apple_gpio_pinctrl_suspend_noirq(struct device *dev) struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); unsigned int i; - if (!pctl->saved || pm_suspend_target_state != PM_SUSPEND_MEM || - !pm_suspend_via_firmware()) + if (pm_suspend_target_state != PM_SUSPEND_MEM || !pm_suspend_via_firmware()) + return 0; + + bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); + /* + * A block that loses every pin: every pin as the hardware has it now, but the ones + * the device tree lists in gpio-reserved-ranges, which may not be written back. The + * pins of the panel's TCON bus, set up by the boot loader, are among them. + */ + pctl->saved_global_valid = false; + if (pctl->restore_all) { + pctl->saved_global = readl(pctl->base + REG_GLOBAL); + pctl->saved_global_valid = true; + if (pctl->cached) + regcache_cache_bypass(pctl->map, true); + for (i = 0; i < pctl->pinctrl_desc.npins; i++) + if (!test_bit(i, pctl->no_restore) && + !regmap_read(pctl->map, REG_GPIO(i), &pctl->saved[i])) + set_bit(i, pctl->saved_valid); + if (pctl->cached) + regcache_cache_bypass(pctl->map, false); return 0; + } /* * Only the pins this driver configured. The others belong to the firmware or another * processor, and writing one back after the wake raises an SError. */ - bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); for_each_set_bit(i, pctl->touched, pctl->pinctrl_desc.npins) if (!regmap_read(pctl->map, REG_GPIO(i), &pctl->saved[i])) set_bit(i, pctl->saved_valid); @@ -694,25 +797,84 @@ static int apple_gpio_pinctrl_resume_noirq(struct device *dev) struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); unsigned int i, val, restored = 0; - if (!pctl->saved || !pm_resume_via_firmware()) + if (!pm_resume_via_firmware()) return 0; + if (pctl->saved_global_valid) { + val = readl(pctl->base + REG_GLOBAL); + writel(pctl->saved_global, pctl->base + REG_GLOBAL); + dev_dbg(dev, "resume: global word %#x, put back to %#x\n", val, + pctl->saved_global); + pctl->saved_global_valid = false; + } + + /* The saved words came from the cache; the hardware is what to compare. */ + if (pctl->cached) + regcache_cache_bypass(pctl->map, true); for_each_set_bit(i, pctl->saved_valid, pctl->pinctrl_desc.npins) { - if (!regmap_read(pctl->map, REG_GPIO(i), &val) && val == pctl->saved[i]) + if (regmap_read(pctl->map, REG_GPIO(i), &val)) + val = 0; + else if (val == pctl->saved[i]) continue; - regmap_write(pctl->map, REG_GPIO(i), pctl->saved[i]); + apple_gpio_safe_reconfigure(pctl, i, val, pctl->saved[i]); restored++; } - dev_dbg(dev, "resume: %u of %u configured pins put back\n", restored, - bitmap_weight(pctl->saved_valid, pctl->pinctrl_desc.npins)); + if (pctl->cached) + regcache_cache_bypass(pctl->map, false); + if (restored) + dev_dbg(dev, "resume: %u of %u configured pins put back\n", restored, + bitmap_weight(pctl->saved_valid, pctl->pinctrl_desc.npins)); bitmap_zero(pctl->saved_valid, pctl->pinctrl_desc.npins); return 0; } +/* + * A resume from disk is a cold boot: the block holds what the boot loader and + * the boot kernel set up, not the restored kernel's pins, while the regmap + * cache still holds the restored kernel's words. Neither an interrupt nor a + * consumer gets its pin back by itself: the IRQ core disables lazily and + * unmasks only an IRQ it recorded as masked, and regmap_update_bits() skips a + * write the cache already has. So an edge interrupt armed before the + * hibernation, such as the HDMI hotplug detect, stays off for good, and an + * output that its consumer sets to the level it had before is never driven. + * Write back, from the cache, every pin this driver set up that the block + * holds differently, in the order a pin takes its word. + * + * The early restore, as a PM domain does not wrap it (its restore_noirq ends + * in the driver's resume_noirq). An uncached block reads back only what the + * boot left, so this does nothing there. + */ +static int apple_gpio_pinctrl_restore_early(struct device *dev) +{ + struct apple_gpio_pinctrl *pctl = dev_get_drvdata(dev); + unsigned int i, cfg, val, restored = 0; + + /* Uncached: no word of the restored kernel to write back. */ + if (!pctl->cached) + return 0; + + for_each_set_bit(i, pctl->touched, pctl->pinctrl_desc.npins) { + if (regmap_read(pctl->map, REG_GPIO(i), &cfg)) + continue; + /* Past the cache, which every write below leaves at @cfg again. */ + val = readl_relaxed(pctl->base + REG_GPIO(i)); + if (val == cfg) + continue; + apple_gpio_safe_reconfigure(pctl, i, val, cfg); + restored++; + } + if (restored) + dev_dbg(dev, "resume from disk: %u of %u configured pins put back\n", restored, + bitmap_weight(pctl->touched, pctl->pinctrl_desc.npins)); + + return 0; +} + static const struct dev_pm_ops apple_gpio_pinctrl_pm_ops = { NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_gpio_pinctrl_suspend_noirq, apple_gpio_pinctrl_resume_noirq) + .restore_early = pm_sleep_ptr(apple_gpio_pinctrl_restore_early), }; static struct platform_driver apple_gpio_pinctrl_driver = { diff --git a/drivers/pmdomain/apple/pmc-test.c b/drivers/pmdomain/apple/pmc-test.c index 4e7f19d47317a..5e0446c8fe21b 100644 --- a/drivers/pmdomain/apple/pmc-test.c +++ b/drivers/pmdomain/apple/pmc-test.c @@ -76,6 +76,23 @@ static void apple_pmc_test_agent_rails(struct kunit *test) rail < 2 ? BIT(1) : 0); } +/* After a resume from disk the enables are written again from the books. */ +static void apple_pmc_test_restore_after_reset(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + unsigned int rail; + + pmc->hw = &apple_pmc_hw_t6040; + KUNIT_ASSERT_EQ(test, apple_pmc_agent_enable(pmc, 1, true), 0); + for (rail = 0; rail < pmc->hw->rails; rail++) + writel(0, pmc->rails + pmc->hw->rail_enable + 4 * rail); + + KUNIT_ASSERT_EQ(test, apple_pmc_restore_state(pmc), 0); + for (rail = 0; rail < pmc->hw->rails; rail++) + KUNIT_EXPECT_EQ(test, apple_pmc_enable_read(pmc, rail), + rail < 2 ? BIT(1) : 0); +} + /* Lifting a DCS limit gives the AP's agent back its own vote, not 0. */ static void apple_pmc_test_dcs_limit_restores_ap_vote(struct kunit *test) { @@ -550,6 +567,7 @@ static struct kunit_case apple_pmc_test_cases[] = { KUNIT_CASE(apple_pmc_test_pending_floor), KUNIT_CASE(apple_pmc_test_agent_enable), KUNIT_CASE(apple_pmc_test_agent_rails), + KUNIT_CASE(apple_pmc_test_restore_after_reset), KUNIT_CASE(apple_pmc_test_dcs_limit_restores_ap_vote), KUNIT_CASE(apple_pmc_test_floor_aggregate), KUNIT_CASE(apple_pmc_test_floor_pending_timeout), diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 586e3f675db9e..2341ff1d0c9fa 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -1534,16 +1534,12 @@ static const struct apple_pmc_hw apple_pmc_hw_t6050 = { * A system sleep that reached the hardware may have taken the enables and floors * with it. Put back what the driver's books say: these are no-ops if they stayed. */ -static int apple_pmc_resume_early(struct device *dev) +static int apple_pmc_restore_state(struct apple_pmc *pmc) { - struct apple_pmc *pmc = dev_get_drvdata(dev); unsigned long flags; unsigned int rail; int ret = 0, err; - if (!pm_resume_via_firmware() || !pmc) - return 0; - raw_spin_lock_irqsave(&pmc->lock, flags); for (rail = 0; rail < pmc->hw->rails; rail++) { err = apple_pmc_enable_write(pmc, rail); @@ -1554,14 +1550,49 @@ static int apple_pmc_resume_early(struct device *dev) } } raw_spin_unlock_irqrestore(&pmc->lock, flags); + + return ret; +} + +static int apple_pmc_resume_early(struct device *dev) +{ + struct apple_pmc *pmc = dev_get_drvdata(dev); + int ret; + + if (!pm_resume_via_firmware() || !pmc) + return 0; + + ret = apple_pmc_restore_state(pmc); if (ret) dev_err(dev, "state not restored after sleep: %d\n", ret); return 0; } +/* + * The kernel restored from a hibernation image runs after a boot loader that + * started from a reset PMC: every agent is off on every rail again, so the + * floors of the agents this driver enabled (the CPU performance controller's + * DCS and fabric votes among them) are no longer honoured. Restore always. + */ +static int apple_pmc_restore_early(struct device *dev) +{ + struct apple_pmc *pmc = dev_get_drvdata(dev); + int ret; + + if (!pmc) + return 0; + + ret = apple_pmc_restore_state(pmc); + if (ret) + dev_err(dev, "state not restored after resume from disk: %d\n", ret); + + return 0; +} + static const struct dev_pm_ops apple_pmc_pm_ops = { - LATE_SYSTEM_SLEEP_PM_OPS(NULL, apple_pmc_resume_early) + .resume_early = pm_sleep_ptr(apple_pmc_resume_early), + .restore_early = pm_sleep_ptr(apple_pmc_restore_early), }; static const struct of_device_id apple_pmc_of_match[] = { diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index d6ac0c2cebd9f..ee7cf19d5816a 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -672,6 +672,46 @@ static int apple_pmgr_ps_resume_noirq(struct device *dev) return 0; } +/* + * After a resume from disk genpd still records the domains as they were when + * the image was taken, but the cold boot behind the resume left the hardware + * in the boot loader's state, and genpd powers on only the domains it recorded + * as off. A domain recorded on but found gated would make its consumers' + * restore touch an unpowered block. Bring each such domain up, parents first. + */ +static void apple_pmgr_ps_restore_hw(struct apple_pmgr_ps *ps, unsigned int depth) +{ + struct gpd_link *link; + u32 reg; + + if (depth > 16) + return; + list_for_each_entry(link, &ps->genpd.child_links, child_node) + if (link->parent->power_on == apple_pmgr_ps_power_on) + apple_pmgr_ps_restore_hw(genpd_to_apple_pmgr_ps(link->parent), depth + 1); + + if (ps->firmware_owned || ps->inherited_on || ps->sleep_driver_owned || + ps->genpd.status != GENPD_STATE_ON) + return; + if (regmap_read(ps->regmap, ps->offset, ®)) + return; + if (FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) == APPLE_PMGR_PS_ACTIVE) + return; + dev_dbg(ps->dev, "PS %s: on in the image, gated after the cold boot (0x%x): powering on\n", + ps->genpd.name, reg); + apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_ACTIVE, + !ps->no_auto_enable && !ps->no_auto_pm); +} + +static int apple_pmgr_ps_restore_noirq(struct device *dev) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + + if (ps) + apple_pmgr_ps_restore_hw(ps, 0); + return 0; +} + static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { struct apple_pmgr_ps *ps = rcdev_to_apple_pmgr_ps(rcdev); @@ -1383,7 +1423,13 @@ MODULE_DEVICE_TABLE(of, apple_pmgr_ps_of_match); static const struct dev_pm_ops apple_pmgr_ps_pm_ops = { LATE_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_late, apple_pmgr_ps_resume_early) - NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_noirq, apple_pmgr_ps_resume_noirq) + .suspend_noirq = pm_sleep_ptr(apple_pmgr_ps_suspend_noirq), + .freeze_noirq = pm_sleep_ptr(apple_pmgr_ps_suspend_noirq), + .poweroff_noirq = pm_sleep_ptr(apple_pmgr_ps_suspend_noirq), + /* Suspend-to-RAM resyncs after the sleep engine; a resume from disk after the cold boot. */ + .resume_noirq = pm_sleep_ptr(apple_pmgr_ps_resume_noirq), + .thaw_noirq = pm_sleep_ptr(apple_pmgr_ps_resume_noirq), + .restore_noirq = pm_sleep_ptr(apple_pmgr_ps_restore_noirq), }; static struct platform_driver apple_pmgr_ps_driver = { diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 7ce01a88bb069..055f6e42760a6 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -447,6 +447,10 @@ static void apple_pmp_report_ready_work(struct work_struct *work) unsigned int ms = rep->polls++ < PMP_REPORT_FAST_POLLS ? PMP_REPORT_POLL_MS : PMP_REPORT_POLL_SLOW_MS; + if (rep->polls == PMP_REPORT_FAST_POLLS + 1) + dev_warn(rep->dev, "PMP still stopped, requests %#llx held\n", + rep->shadow); + queue_delayed_work(system_freezable_power_efficient_wq, &rep->ready_work, msecs_to_jiffies(ms)); return; @@ -559,6 +563,90 @@ void apple_pmp_report_wake(struct device *pmp, bool lost_power) } EXPORT_SYMBOL_GPL(apple_pmp_report_wake); +/* + * Hold requests again for a PMP that stopped behind this driver's back: on + * a resume from hibernation the boot loader has cold-restarted the PMP + * (halted, its report region reset when it starts) while the restored image + * still has requests handed. Requests made from now on stay in @shadow and + * the ready work hands all of them over once the PMP runs, as at probe. + * Returns whether the requests are held now. Called with @rep->lock held. + */ +static bool apple_pmp_report_hold_locked(struct apple_pmp_report *rep) +{ + if (rep->defer) + return true; + if (!rep->offsets->defer_requests || apple_pmp_report_running(rep)) + return false; + rep->defer = true; + rep->polls = 0; + queue_delayed_work(system_freezable_power_efficient_wq, &rep->ready_work, 0); + return true; +} + +static bool apple_pmp_report_handed(struct apple_pmp_report *rep) +{ + unsigned long flags; + bool handed; + u64 want; + + spin_lock_irqsave(&rep->lock, flags); + handed = !rep->defer; + want = rep->shadow; + spin_unlock_irqrestore(&rep->lock, flags); + return handed && (readq(rep->base + rep->offsets->actual) & want) == want; +} + +/** + * apple_pmp_report_wait_handed() - wait until every held request is acknowledged + * @timeout_ms: upper bound of the wait + * + * For a consumer that restarts its own firmware after a resume from + * hibernation and must not run it before the PMP has taken the power state + * requests made for it (the GPU's pmp-gfx request). Sleeps. + * + * Return: 0 once every report runs with its requests acknowledged, or + * -ETIMEDOUT. + */ +int apple_pmp_report_wait_handed(unsigned int timeout_ms) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); + struct apple_pmp_report *rep; + bool done; + + for (;;) { + done = true; + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) + done &= apple_pmp_report_handed(rep); + rcu_read_unlock(); + if (done) + return 0; + if (time_after(jiffies, deadline)) + return -ETIMEDOUT; + msleep(10); + } +} +EXPORT_SYMBOL_GPL(apple_pmp_report_wait_handed); + +static int apple_pmp_report_restore_noirq(struct device *dev) +{ + struct apple_pmp_report *rep = dev_get_drvdata(dev); + unsigned long flags; + bool held; + u64 want; + + if (!rep) + return 0; + spin_lock_irqsave(&rep->lock, flags); + held = !rep->defer && apple_pmp_report_hold_locked(rep); + want = rep->shadow; + spin_unlock_irqrestore(&rep->lock, flags); + if (held) + dev_info(dev, "PMP stopped after restore: requests %#llx held until it runs\n", + want); + return 0; +} + /* The unique, enabled and acknowledged T6030 display request (DISP, reg 7). */ static struct device_node *apple_pmp_report_t6030_entry(struct device_node *report) { @@ -2269,8 +2357,10 @@ static int apple_pmp_report_resume(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_pmp_report_pm_ops, apple_pmp_report_suspend, - apple_pmp_report_resume); +static const struct dev_pm_ops apple_pmp_report_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_pmp_report_suspend, apple_pmp_report_resume) + .restore_noirq = apple_pmp_report_restore_noirq, +}; static int apple_pmp_report_probe(struct platform_device *pdev) { @@ -2555,7 +2645,8 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, rep->shadow |= bit_val; else rep->shadow &= ~bit_val; - ready = !rep->defer; + /* Never write a request to a stopped PMP (see the hold helper). */ + ready = !apple_pmp_report_hold_locked(rep); if (ready) { val = readq(rep->base + rep->offsets->tgt_read); val &= ~bit_val; diff --git a/drivers/power/reset/macsmc-reboot.c b/drivers/power/reset/macsmc-reboot.c index a2e1422e05bd8..de8161a96eabe 100644 --- a/drivers/power/reset/macsmc-reboot.c +++ b/drivers/power/reset/macsmc-reboot.c @@ -127,15 +127,33 @@ static int macsmc_prepare_atomic(struct sys_off_data *data) static int macsmc_power_off(struct sys_off_data *data) { struct macsmc_reboot *reboot = data->cb_data; + int ret, i; dev_info(reboot->dev, "Issuing power off (off1)\n"); - if (apple_smc_write_u32_atomic(reboot->smc, SMC_KEY(MBSE), SMC_KEY(off1)) < 0) { - dev_err(reboot->dev, "Failed to issue MBSE = off1 (power_off)\n"); - } else { + /* + * This is the only power-off path when Linux runs at EL2 without a + * monitor, and where a PSCI SYSTEM_OFF follows, a monitor without a + * power-off path resets the machine instead. So a slow or refused + * off1 must not fall through after only 100 ms: retry a refused + * command and give an accepted one time to take effect, and say on + * the console which of the two happened. + */ + for (i = 0; i < 3; i++) { + ret = apple_smc_write_u32_atomic(reboot->smc, SMC_KEY(MBSE), SMC_KEY(off1)); + if (ret >= 0) + break; + dev_warn(reboot->dev, "MBSE = off1 refused (%d), attempt %d\n", ret, i + 1); mdelay(100); - WARN_ONCE(1, "Unable to power off system\n"); } + if (ret < 0) { + dev_err(reboot->dev, "Failed to issue MBSE = off1 (power_off)\n"); + return NOTIFY_OK; + } + dev_dbg(reboot->dev, "off1 accepted, waiting for the power to drop\n"); + for (i = 0; i < 50; i++) + mdelay(100); + WARN_ONCE(1, "Unable to power off system: off1 accepted, power still on after 5 s\n"); return NOTIFY_OK; } diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index cd46c9b9a10b1..2e07c1a393371 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -432,17 +432,59 @@ static int apple_pmu_rtc_prepare(struct device *dev) if (rtc->io_failed) return -EBUSY; /* + * Suspend to idle keeps the PMU powered and the alarm wakes the system. * A firmware-driven suspend programs the wake sources itself, and the - * alarm stays armed in the PMU, which stays powered. Only a sleep state - * without either has no way to bring the alarm back. + * alarm stays armed in the PMU, which stays powered. Hibernation (the + * target state stays PM_SUSPEND_ON) is accepted as well: the restore + * below has the alarm set again. Only a sleep state without any of + * these has no way to bring the alarm back. */ - if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && !pm_suspend_via_firmware()) + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && + pm_suspend_target_state != PM_SUSPEND_ON && !pm_suspend_via_firmware()) return -EBUSY; return 0; } +/* + * Only a hibernation that powers off through platform hibernation ops gets + * here; a plain power-off or reboot after the image leaves the alarm as it is. + * Whether an armed PMU alarm powers the machine on again after the + * hibernation power-off is not established, so do not leave one armed. + */ +static int apple_pmu_rtc_poweroff(struct device *dev) +{ + struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); + int error; + + mutex_lock(&rtc->transaction); + error = apple_pmu_rtc_stop_alarm(rtc); + mutex_unlock(&rtc->transaction); + if (error) + dev_warn(dev, "could not stop the alarm before power-off: %d\n", error); + return 0; +} + +/* + * A resume from disk went through a boot. The firmware and the kernel that + * loaded the image had the PMU in between: either may have stopped the alarm + * or set the comparator anew, and the alarm's time may have passed with the + * machine off, its event acknowledged by one of them. The RTC core still + * counts the alarm as set. Have it go through its timers: an alarm that is due + * is reported, a later one is written to the PMU again, and with none set the + * PMU's alarm is stopped. + */ +static int apple_pmu_rtc_restore(struct device *dev) +{ + struct apple_pmu_rtc *rtc = dev_get_drvdata(dev); + + rtc_update_irq(rtc->clock, 1, RTC_IRQF | RTC_AF); + return 0; +} + static const struct dev_pm_ops apple_pmu_rtc_pm = { .prepare = apple_pmu_rtc_prepare, + .poweroff = apple_pmu_rtc_poweroff, + .restore = apple_pmu_rtc_restore, }; static void apple_pmu_rtc_release(void *context) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 9225b054e7cde..ef61ebcc97f61 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -98,6 +98,7 @@ config APPLE_SOC_SLEEP depends on ARM64 && (ARCH_APPLE || COMPILE_TEST) depends on HOTPLUG_CPU && SUSPEND select APPLE_TUNABLE + select MFD_SYSCON if MFD_MACSMC=y help Powers the secondary CPUs off and suspends the system to RAM on an Apple SoC whose kernel runs directly at EL2, with no monitor to ask. @@ -105,8 +106,25 @@ config APPLE_SOC_SLEEP /chosen/aurora,s2r-mailbox. Only SoCs whose sleep has been measured and tested are supported, - each by its match data: today T8140 and T8132. On every other SoC - the driver doesn't bind, and s2idle stays the only system sleep. + each by its match data: today T8140, T8132, and with MFD_MACSMC + built in the T6040. On every other SoC the driver doesn't bind, and + s2idle stays the only system sleep. + + The T6040 (M4 Pro) part has its own sequences and refuses first, on + the boards whose device tree enables its sleep controller: its + suspend handler lists every blocker in the live state (EL2, the boot + object's return path, the CPUs that can park, the display power + states that have to be off before a sleep). Unless + t6040_sleep.allow_entry=1 is given and no blocker remains, a request + for suspend-to-RAM prints the blockers and enters nothing, and + suspend-to-idle stays the default sleep. With it the SoC is powered + down to suspend-to-RAM and comes back through the boot object; the + AON watchdog is the net over the whole cycle + (t6040_sleep.sleep_wd_secs at the power-down itself, + t6040_sleep.wd2_secs over a dry run). With t6040_sleep.dry_run=1 + every driver's firmware-driven sleep step that can be taken back + runs, the CPUs park and are held, and the sleep is refused at the + commit point and resumed warm. config APPLE_SOC_SLEEP_KUNIT_TEST bool "KUnit tests for the Apple SoC sleep match data" if !KUNIT_ALL_TESTS @@ -320,6 +338,19 @@ config APPLE_CLPC Say 'y' or 'm' here if you have an Apple SoC with a PMC. +config APPLE_AURA_HIBERNATE + def_bool y + depends on ARCH_APPLE && HIBERNATION && OF_EARLY_FLATTREE + help + Under aura, the boot loader and the firmware keep memory alive + below Linux's allocations, and on Apple M4 Pro boots the boot + firmware places it at a different address every time. With + aura's fixed layout (/chosen/aura,hibernate-layout) this keeps + every reserved range out of the hibernation image; without it, + hibernation and resume are refused instead of overwriting the + running boot loader. The refusal is part of the hibernation + support of these machines, so the option cannot be turned off. + config APPLE_DISPLAY_GATE bool "Apple internal display enable gate" depends on ARCH_APPLE && OF_DYNAMIC && OF_OVERLAY diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 5065bb7a1324a..6f3f35604eb95 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -14,10 +14,14 @@ obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o obj-$(CONFIG_APPLE_CLUSTER_PWRGATE) += apple-cluster-pwrgate.o obj-$(CONFIG_APPLE_SOC_SLEEP) += apple-sleep.o -apple-sleep-y = sleep.o +apple-sleep-y = sleep.o sleep-registry.o +# The T6040 sleep calls the SMC driver directly +ifeq ($(CONFIG_MFD_MACSMC),y) +apple-sleep-y += sleep-t6040.o +endif obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o -apple-rtkit-y = rtkit.o rtkit-crashlog.o +apple-rtkit-y = rtkit.o rtkit-crashlog.o rtkit-hib-handoff.o obj-$(CONFIG_APPLE_RTKIT_HELPER) += apple-rtkit-helper.o apple-rtkit-helper-y = rtkit-helper.o @@ -42,6 +46,8 @@ DTC_FLAGS_t6030-gate-test := -Wno-interrupts_property -Wno-chosen_node_is_root \ DTC_FLAGS_t8122-gate-test := -Wno-interrupts_property -Wno-chosen_node_is_root DTC_FLAGS_t6041-gate-test := -Wno-unit_address_vs_reg -Wno-avoid_unnecessary_addr_size +obj-$(CONFIG_APPLE_AURA_HIBERNATE) += aura-hibernate.o + obj-$(CONFIG_APPLE_TUNABLE) += apple-tunable.o apple-tunable-y = tunable.o diff --git a/drivers/soc/apple/aura-hibernate.c b/drivers/soc/apple/aura-hibernate.c new file mode 100644 index 0000000000000..8c50b3cd33dba --- /dev/null +++ b/drivers/soc/apple/aura-hibernate.c @@ -0,0 +1,546 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Hibernation on Apple Silicon under an aura boot object. + * + * A hibernation image restores every saved page to the frame it was taken from, and swsusp saves + * reserved pages as long as they are mapped. Under aura the memory below Linux's own allocations + * holds what the boot loader keeps alive across the hand-off: aura itself (resident at EL2), its + * heap with the firmware buffers it handed over, the device tree, the initramfs, and the ranges + * the boot firmware placed for the co-processors. Restoring old contents there would overwrite + * the running boot loader and the live firmware state of the resuming boot. + * + * aura marks a boot whose layout is the same on every boot with /chosen/aura,hibernate-layout + * (start, end of the range it keeps from Linux). On such a boot every /memreserve/ range and every + * non-reusable /reserved-memory range is registered as nosave, except the kernel image itself. + * Without the property, hibernation and resume are refused, since the reserved ranges move. + * + * The boot firmware still places some firmware segments anew on every boot (the AOP __OS_LOG), + * inside the range aura keeps from Linux. A kernel restored from an image knows the device tree of the boot + * it was started on, not of the boot that resumed it. aura therefore hands Linux its device tree + * at a fixed address inside that range (/chosen/aura,hibernate-dt): the image does not save it, + * so after a resume the kernel finds this boot's tree at the address it was started with, and + * aura_hibernate_boot_fw_regions() reads a device's firmware regions from it. + * + * aura itself moves with the boot object, and with it its S2R mailbox: a restored kernel reads the + * mailbox of the boot it runs on from that tree too (aura_hibernate_boot_chosen_u64()). + */ + +#define pr_fmt(fmt) "aura-hibernate: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool aura_layout_fixed; +static u64 aura_layout[2]; +/* Where aura puts the device tree on every boot; 0 when it does not. */ +static phys_addr_t aura_boot_dt; +#define AURA_BOOT_DT_MAX SZ_2M + +/* + * Ranges kept out of the image, in page frames. Filled once at late_initcall, + * read through the arm64 hibernation platform hooks. A static table avoids + * register_nosave_region(), which allocates from memblock and is meant for + * setup_arch() time. + */ +#define AURA_NOSAVE_MAX 128 +static struct { unsigned long start, end; } aura_nosave[AURA_NOSAVE_MAX]; +static unsigned int aura_nosave_count; + +/* + * Apple silicon Macs run firmware on their coprocessors (storage, secure + * enclave, display, always-on processor, system controller, GPU and more). + * Restoring an image means a cold boot through the platform boot chain, which + * restarts that firmware, but most of the Linux drivers have no + * freeze/poweroff/restore path that rebuilds their mailbox, ring and DMA state + * from a cold start. The restored kernel would keep using stale state, so + * writing and restoring images stays disabled on these machines, except where + * the boot object gives the OS a fixed memory layout and the drivers rebuild + * their coprocessor state from the cold boot that restores the image (aura + * hib=1, see above). + */ +static bool aura_hibernate_blocked(void) +{ + return of_machine_is_compatible("apple,arm-platform") && !aura_layout_fixed; +} + +static bool aura_hibernate_pfn_nosave(unsigned long pfn) +{ + unsigned int i; + + for (i = 0; i < aura_nosave_count; i++) + if (pfn >= aura_nosave[i].start && pfn < aura_nosave[i].end) + return true; + return false; +} + +/* Whether the ranges kept out of the image cover all of [start, end). */ +static bool aura_hibernate_range_nosave(u64 start, u64 end) +{ + unsigned long pfn = PFN_DOWN(start), last = PFN_UP(end); + unsigned int i; + bool moved; + + while (pfn < last) { + moved = false; + for (i = 0; i < aura_nosave_count; i++) { + if (pfn >= aura_nosave[i].start && pfn < aura_nosave[i].end) { + pfn = aura_nosave[i].end; + moved = true; + } + } + if (!moved) + return false; + } + return true; +} + +/* + * The image restores every page to the frame it came from, so the boot that + * resumes it must give the kernel what the boot that wrote it did: on Apple + * machines at EL2 the same CTXR exec window (secondaries only execute inside + * it) and aura's fixed hibernation layout. They are added to the layout cookie + * of arm64 hibernation. + * + * aura's own image moves with the boot object, and its CPU release addresses + * and S2R mailbox with it, inside the range the layout keeps from Linux. The + * restored kernel uses neither address of the boot that wrote the image: its + * CPUs come back from the boot kernel's pen, a CPU started at its reset vector + * again finds the release target the boot kernel wrote (both kernels run at the + * window's fixed address), and the T6040 sleep driver takes this boot's mailbox + * from aura's tree after the restore. What has to match is the spin table's + * shape (the same CPUs, at the same offsets from the mailbox) and that this + * boot keeps all of it out of the image. + */ +static u64 aura_hibernate_layout(u64 h, struct arm64_hibernate_layout *lay) +{ + static const char * const chosen_props[] = { + "aura,hibernate-layout", "aura,hibernate-dt", + "aurora,ctxr-exec-window", + }; + u64 mbox = 0, layout[2], win[2]; + struct device_node *np; + const void *v; + u8 kept; + int i, len; + + np = of_find_node_by_path("/chosen"); + for (i = 0; np && i < ARRAY_SIZE(chosen_props); i++) { + v = of_get_property(np, chosen_props[i], &len); + if (!v) + continue; + h = arm64_hibernate_layout_mix(h, chosen_props[i], strlen(chosen_props[i])); + h = arm64_hibernate_layout_mix(h, v, len); + } + if (np && !of_property_read_u64(np, "aurora,s2r-mailbox", &mbox)) { + lay->have_base = true; + lay->base = mbox; + kept = aura_hibernate_pfn_nosave(PHYS_PFN(mbox)); + h = arm64_hibernate_layout_mix(h, "aurora,s2r-mailbox", + strlen("aurora,s2r-mailbox")); + h = arm64_hibernate_layout_mix(h, &kept, sizeof(kept)); + } + /* aura's layout less the OS window, only where all of it is kept out of the image */ + if (np && !of_property_read_u64_array(np, "aura,hibernate-layout", layout, 2) && + !of_property_read_u64_array(np, "aurora,ctxr-exec-window", win, 2) && + layout[0] <= win[0] && win[0] + win[1] <= layout[1]) { + lay->ex[lay->nex][0] = layout[0]; + lay->ex[lay->nex][1] = win[0]; + if (aura_hibernate_range_nosave(lay->ex[lay->nex][0], lay->ex[lay->nex][1])) + lay->nex++; + lay->ex[lay->nex][0] = win[0] + win[1]; + lay->ex[lay->nex][1] = layout[1]; + if (aura_hibernate_range_nosave(lay->ex[lay->nex][0], lay->ex[lay->nex][1])) + lay->nex++; + } + of_node_put(np); + return h; +} + +static const struct arm64_hibernate_platform aura_hibernate_platform = { + .pfn_nosave = aura_hibernate_pfn_nosave, + .layout = aura_hibernate_layout, +}; + +static void __init aura_nosave_add(unsigned long s, unsigned long e) +{ + if (s >= e) + return; + if (aura_nosave_count == AURA_NOSAVE_MAX) { + pr_warn("more than %d reserved ranges; hibernation stays disabled\n", AURA_NOSAVE_MAX); + aura_nosave_count++; + return; + } + if (aura_nosave_count < AURA_NOSAVE_MAX) { + aura_nosave[aura_nosave_count].start = s; + aura_nosave[aura_nosave_count].end = e; + aura_nosave_count++; + } +} + +static void __init aura_nosave_range(u64 start, u64 end, unsigned long kstart, unsigned long kend, + const char *what) +{ + unsigned long s = PFN_DOWN(start), e = PFN_UP(end); + + if (s >= e) + return; + /* The kernel image is Linux's own and is saved; split the range around it. */ + if (s < kend && e > kstart) { + if (s < kstart) + aura_nosave_add(s, kstart); + if (e > kend) + aura_nosave_add(kend, e); + } else { + aura_nosave_add(s, e); + } + pr_debug("nosave %s [%#llx, %#llx)\n", what, start, end); +} + +static bool aura_fw_region_kept(u64 start, u64 size) +{ + u64 end = start + size, a; + + if (end < start) + return false; + /* + * Inside the range aura keeps from Linux below its device tree slot, where the boot firmware places the + * firmware: reserved, and not saved in the image. Not the slot or aura's heap above it. + */ + if (start >= aura_layout[0] && end <= aura_boot_dt) + return true; + /* Otherwise Linux must neither own nor map any of it (no-map or not RAM). */ + for (a = start & PAGE_MASK; a < end; a += PAGE_SIZE) + if (memblock_is_map_memory(a)) + return false; + return true; +} + +/* + * Whether @fdt, mapped from @slot, is a tree aura wrote there: a valid header that fits the slot, and + * a /chosen naming this slot and the fixed layout @layout. + */ +static bool aura_slot_tree_valid(const void *fdt, phys_addr_t slot, const u64 layout[2]) +{ + const fdt64_t *v; + int node, len; + + if (fdt_check_header(fdt) || fdt_totalsize(fdt) > AURA_BOOT_DT_MAX) + return false; + node = fdt_path_offset(fdt, "/chosen"); + if (node < 0) + return false; + v = fdt_getprop(fdt, node, "aura,hibernate-dt", &len); + if (!v || len != sizeof(*v) || fdt64_ld(v) != slot) + return false; + v = fdt_getprop(fdt, node, "aura,hibernate-layout", &len); + return v && len == 2 * sizeof(*v) && fdt64_ld(&v[0]) == layout[0] && + fdt64_ld(&v[1]) == layout[1]; +} + +/* + * Reads the firmware memory-regions of @np (its memory-region phandles, each + * region's reg and the iommu-addresses entry naming @np) from the device tree + * of the running boot, which after a resume from disk is not the one the + * kernel was started with. Returns the number of regions, at most @max, or a + * negative errno: -ENODEV when aura does not hand the tree over at a fixed + * address. Only flat, untranslated layouts are accepted: @np's parent without + * dma-ranges, a /reserved-memory without a non-empty ranges. + */ +int aura_hibernate_boot_fw_regions(struct device_node *np, struct aura_fw_region *out, + unsigned int max) +{ + const fdt32_t *phandles, *p; + unsigned int n = 0; + int node, parent, len, ret, i; + u32 own; + char *path; + void *fdt; + + if (!aura_boot_dt || !np) + return -ENODEV; + fdt = memremap(aura_boot_dt, AURA_BOOT_DT_MAX, MEMREMAP_WB); + if (!fdt) + return -ENOMEM; + ret = -EINVAL; + /* The tree aura handed over on this boot, with the same fixed layout. */ + if (!aura_slot_tree_valid(fdt, aura_boot_dt, aura_layout)) + goto out; + + path = kasprintf(GFP_KERNEL, "%pOF", np); + if (!path) { + ret = -ENOMEM; + goto out; + } + node = fdt_path_offset(fdt, path); + kfree(path); + ret = -ENOENT; + if (node < 0) + goto out; + ret = -EINVAL; + own = fdt_get_phandle(fdt, node); + parent = fdt_parent_offset(fdt, node); + if (!own || parent < 0 || fdt_getprop(fdt, parent, "dma-ranges", NULL)) + goto out; + phandles = fdt_getprop(fdt, node, "memory-region", &len); + if (!phandles || len <= 0 || len % sizeof(*phandles)) + goto out; + + for (i = 0; i < len / (int)sizeof(*phandles); i++) { + int region, rparent, ac, sc, cells, pos, plen; + const fdt32_t *reg, *iommu; + u64 phys, size, iova = 0, length = 0; + bool found = false; + + region = fdt_node_offset_by_phandle(fdt, fdt32_to_cpu(phandles[i])); + if (region < 0) + goto out; + rparent = fdt_parent_offset(fdt, region); + if (rparent < 0) + goto out; + p = fdt_getprop(fdt, rparent, "ranges", &plen); + if (p && plen) + goto out; + ac = fdt_address_cells(fdt, rparent); + sc = fdt_size_cells(fdt, rparent); + reg = fdt_getprop(fdt, region, "reg", &plen); + if (ac < 1 || ac > 2 || sc < 1 || sc > 2 || !reg || + plen < (int)((ac + sc) * sizeof(*reg))) + goto out; + phys = fdt32_to_cpu(reg[0]); + if (ac == 2) + phys = phys << 32 | fdt32_to_cpu(reg[1]); + size = fdt32_to_cpu(reg[ac]); + if (sc == 2) + size = size << 32 | fdt32_to_cpu(reg[ac + 1]); + + /* Entries: owner phandle, then an address and a size in the owner's bus cells. */ + iommu = fdt_getprop(fdt, region, "iommu-addresses", &plen); + if (!iommu || plen <= 0 || plen % sizeof(*iommu)) + goto out; + cells = plen / sizeof(*iommu); + for (pos = 0; pos < cells;) { + u32 ph = fdt32_to_cpu(iommu[pos++]); + int owner = fdt_node_offset_by_phandle(fdt, ph), op, oac, osc; + + if (owner < 0) + goto out; + op = fdt_parent_offset(fdt, owner); + if (op < 0) + goto out; + oac = fdt_address_cells(fdt, op); + osc = fdt_size_cells(fdt, op); + if (oac < 1 || oac > 2 || osc < 1 || osc > 2 || oac + osc > cells - pos) + goto out; + if (ph == own) { + if (found) + goto out; + found = true; + iova = fdt32_to_cpu(iommu[pos]); + if (oac == 2) + iova = iova << 32 | fdt32_to_cpu(iommu[pos + 1]); + length = fdt32_to_cpu(iommu[pos + oac]); + if (osc == 2) + length = length << 32 | fdt32_to_cpu(iommu[pos + oac + 1]); + } + pos += oac + osc; + } + if (!found || length != size || !size) + goto out; + if (!aura_fw_region_kept(phys, size)) { + pr_warn("%pOF: this boot's firmware region [%#llx, +%#llx) is memory Linux owns\n", + np, phys, size); + ret = -EBUSY; + goto out; + } + if (n == max) { + ret = -E2BIG; + goto out; + } + out[n].phys = phys; + out[n].iova = iova; + out[n].size = size; + n++; + } + ret = n; +out: + memunmap(fdt); + return ret; +} +EXPORT_SYMBOL_GPL(aura_hibernate_boot_fw_regions); + +/* + * Reads the u64 property @name of /chosen from the device tree of the running boot, which after a + * resume from disk is not the one the kernel was started with. Returns 0, -ENODEV when aura does + * not hand the tree over at a fixed address, -EINVAL when the slot holds no tree for this layout, + * -ENOENT when the tree has no such property. + */ +int aura_hibernate_boot_chosen_u64(const char *name, u64 *val) +{ + const fdt64_t *v; + int node, len, ret; + void *fdt; + + if (!aura_boot_dt) + return -ENODEV; + fdt = memremap(aura_boot_dt, AURA_BOOT_DT_MAX, MEMREMAP_WB); + if (!fdt) + return -ENOMEM; + ret = -EINVAL; + if (!aura_slot_tree_valid(fdt, aura_boot_dt, aura_layout)) + goto out; + ret = -ENOENT; + node = fdt_path_offset(fdt, "/chosen"); + v = node < 0 ? NULL : fdt_getprop(fdt, node, name, &len); + if (!v || len != sizeof(*v)) + goto out; + *val = fdt64_ld(v); + ret = 0; +out: + memunmap(fdt); + return ret; +} +EXPORT_SYMBOL_GPL(aura_hibernate_boot_chosen_u64); + +struct aura_hibernate_reprobe { + struct work_struct work; + struct device *dev; +}; + +static void aura_hibernate_reprobe_work(struct work_struct *work) +{ + struct aura_hibernate_reprobe *rp = container_of(work, struct aura_hibernate_reprobe, work); + int ret; + + ret = device_reprobe(rp->dev); + /* A failed probe is not reported by device_reprobe() itself. */ + if (ret || !READ_ONCE(rp->dev->driver)) + dev_err(rp->dev, "probe after the resume from disk failed: %d\n", ret); + put_device(rp->dev); + kfree(rp); +} + +/* + * Unbinds @dev and probes it again once the tasks are thawed, for a driver whose co-processor + * the boot that resumed the image left halted: its unbind and its probe know how to drop a + * session that died under it and start the firmware anew, as at boot, and a restore callback + * cannot run either. The work runs on the freezable workqueue, which the restore finds frozen, + * so it starts after every device's restore and its complete step, when probing is allowed + * again. Built in, so that a driver module going away meanwhile leaves nothing behind. + */ +int aura_hibernate_reprobe_after_thaw(struct device *dev) +{ + struct aura_hibernate_reprobe *rp; + + rp = kzalloc_obj(*rp); + if (!rp) + return -ENOMEM; + INIT_WORK(&rp->work, aura_hibernate_reprobe_work); + rp->dev = get_device(dev); + queue_work(system_freezable_wq, &rp->work); + return 0; +} +EXPORT_SYMBOL_GPL(aura_hibernate_reprobe_after_thaw); + +static int __init aura_hibernate_init(void) +{ + unsigned long kstart = PFN_DOWN(__pa_symbol(_text)), kend = PFN_UP(__pa_symbol(_end)); + struct device_node *chosen, *rmem, *child; + unsigned int count = 0; + const void *fdt = initial_boot_params; + u64 layout[2], dt_slot; + int i, n; + + hibernation_set_platform_check(aura_hibernate_blocked, + "coprocessor firmware state is not restored from an image"); + arm64_hibernate_set_platform(&aura_hibernate_platform); + + /* Only a boot loader that keeps the layout fixed publishes it. */ + chosen = of_find_node_by_path("/chosen"); + if (!chosen || + of_property_read_u64_array(chosen, "aura,hibernate-layout", layout, 2)) { + of_node_put(chosen); + return 0; + } + /* + * The device tree slot. Trusted where this kernel's tree is, or, when a later stage (U-Boot and + * the EFI stub) handed the kernel its own copy of aura's tree, where the slot still holds aura's + * tree for this layout: those stages do not move the firmware segments, so aura's tree and the + * kernel's copy agree on them. + */ + if (!of_property_read_u64(chosen, "aura,hibernate-dt", &dt_slot)) { + bool in_layout = dt_slot >= layout[0] && dt_slot + AURA_BOOT_DT_MAX <= layout[1]; + void *slot_fdt; + + if (in_layout && dt_slot == __fdt_pointer) { + aura_boot_dt = dt_slot; + } else if (in_layout) { + slot_fdt = memremap(dt_slot, AURA_BOOT_DT_MAX, MEMREMAP_WB); + if (slot_fdt && aura_slot_tree_valid(slot_fdt, dt_slot, layout)) { + aura_boot_dt = dt_slot; + pr_info("device tree handed over at %pa; using aura's tree at %#llx\n", + &__fdt_pointer, dt_slot); + } else { + pr_warn("device tree at %pa, aura's slot %#llx holds no tree for this layout\n", + &__fdt_pointer, dt_slot); + } + if (slot_fdt) + memunmap(slot_fdt); + } else { + pr_warn("device tree slot %#llx outside the fixed layout\n", dt_slot); + } + } + of_node_put(chosen); + + if (fdt) { + n = fdt_num_mem_rsv(fdt); + for (i = 0; i < n; i++) { + u64 addr, size; + + if (fdt_get_mem_rsv(fdt, i, &addr, &size) || !size) + continue; + aura_nosave_range(addr, addr + size, kstart, kend, "memreserve"); + count++; + } + } + + rmem = of_find_node_by_path("/reserved-memory"); + for_each_available_child_of_node(rmem, child) { + struct resource r; + + if (of_property_read_bool(child, "reusable")) + continue; + if (of_address_to_resource(child, 0, &r)) + continue; + aura_nosave_range(r.start, r.end + 1, kstart, kend, "reserved-memory"); + count++; + } + of_node_put(rmem); + + if (aura_nosave_count > AURA_NOSAVE_MAX) { + aura_nosave_count = 0; + return 0; + } + aura_layout[0] = layout[0]; + aura_layout[1] = layout[1]; + aura_layout_fixed = true; + pr_info("fixed boot layout [%#llx, %#llx), %u reserved ranges kept out of the image%s\n", + layout[0], layout[1], count, + aura_boot_dt ? ", device tree at a fixed address" : ""); + return 0; +} +late_initcall(aura_hibernate_init); diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index c1d415c0bd744..23c371ec7a9d4 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -1,12 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * System sleep for the SEP driver: the notifier only forwards "about to sleep" - * and "back from sleep" to Rust, which ends any Touch ID capture before the - * system suspends (see pm.rs), and the enclave's power domains are kept on - * across sleep. The device's suspend and resume callbacks are the firmware - * suspend's: the enclave is told, and gates its own domain, whose state word is - * read here. On T8140 at EL2 the sleep driver sends the request at the - * irreversible commit boundary. Only errors before send may refuse. + * System sleep for the SEP driver: the notifier forwards the sleep events to + * Rust, which ends any Touch ID capture before the system suspends (see + * pm.rs); after a resume from disk on a cold boot it probes the device again; + * and the enclave's power domains are kept on across sleep. The device's + * suspend and resume callbacks are the firmware suspend's: the enclave is + * told, and gates its own domain, whose state word is read here. On T8140 at + * EL2 the sleep driver sends the request at the irreversible commit boundary. + * Only errors before send may refuse. */ #include #include @@ -23,9 +24,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -42,33 +45,128 @@ module_param(sleep_refuse_before_send, bool, 0644); MODULE_PARM_DESC(sleep_refuse_before_send, "Test only: refuse the next prepared deep sleep before sending SEP SLEEP"); -static void (*sep_pm_event)(bool entering); +static int (*sep_pm_event)(int event); +static struct device *sep_pm_dev; static int (*sep_pm_sleep)(void); static void (*sep_pm_wake)(void); static bool (*sep_pm_told)(void); /* In the device's .suspend, the only caller that can leave the request to the sleep. */ static bool sep_pm_suspending; +/* The marker dt.rs leaves on the SEP node once the registration is sent. */ +#define SEP_REGISTERED_PROP "apple,sep-shmem-registered-iova" + +/* + * A resume from disk on a cold boot: the enclave runs a freshly booted sepOS + * that has never seen this kernel's registration, so the restored session is + * dead. The device is probed again once the tasks are thawed, as at boot: + * probe sends a new registration (the marker is gone with the session it + * guarded), opens the xART store and loads the host state from disk again (the + * boot that resumed this image may have changed both), and recreates + * /dev/sep-bio. + */ +static void sep_pm_schedule_reprobe(struct device *dev) +{ + struct device_node *np = dev->of_node; + struct property *prop; + int ret; + + /* + * Only here, with the cold boot established: removing the marker on a + * live enclave would let a second registration park it. + */ + prop = np ? of_find_property(np, SEP_REGISTERED_PROP, NULL) : NULL; + if (prop && of_remove_property(np, prop)) { + dev_err(dev, "resume from disk: the registration marker could not be removed; Touch ID stays down until a reboot\n"); + return; + } + + if (!IS_ENABLED(CONFIG_APPLE_AURA_HIBERNATE)) { + dev_err(dev, "resume from disk: no aura hibernation support to probe the SEP again; Touch ID stays down until a reboot\n"); + return; + } + + dev_info(dev, "resume from disk: attaching the restarted enclave again\n"); + ret = aura_hibernate_reprobe_after_thaw(dev); + if (ret) + dev_err(dev, "resume from disk: probing the SEP again could not be queued (%d); Touch ID stays down until a reboot\n", + ret); +} + static int sep_pm_notify(struct notifier_block *nb, unsigned long action, void *data) { - void (*event)(bool entering) = READ_ONCE(sep_pm_event); + int (*event)(int event) = READ_ONCE(sep_pm_event); + struct device *dev = READ_ONCE(sep_pm_dev); + int cold; if (!event) return NOTIFY_DONE; switch (action) { case PM_SUSPEND_PREPARE: + sep_diag_sleep(true); + event(SEP_PM_SLEEP); + return NOTIFY_OK; + case PM_POST_SUSPEND: + event(SEP_PM_WAKE); + sep_diag_sleep(false); + return NOTIFY_OK; case PM_HIBERNATION_PREPARE: + sep_diag_sleep(true); + event(SEP_PM_HIBERNATE); + return NOTIFY_OK; case PM_RESTORE_PREPARE: + /* + * This kernel is about to load an image. If it registered with the + * enclave, the enclave keeps that session and may DMA into memory + * the image overwrites, and the restored kernel would register a + * second time: refuse the restore (this boot goes on instead). + */ + if (event(SEP_PM_RESTORE) > 0) { + pr_err("apple_sep: this kernel attached the SEP; refusing to restore a hibernation image over it\n"); + return NOTIFY_BAD; + } + /* Not before the refusal: no PM_POST_RESTORE comes to the one who refused. */ sep_diag_sleep(true); - event(true); return NOTIFY_OK; - case PM_POST_SUSPEND: - case PM_POST_HIBERNATION: case PM_POST_RESTORE: - event(false); + event(SEP_PM_HIBERNATE_WARM); + sep_diag_sleep(false); + return NOTIFY_OK; + case PM_POST_HIBERNATION: sep_diag_sleep(false); + /* + * Ends an aborted hibernation, the wake of a hybrid sleep and a + * resume from the image alike. iBoot boots sepOS anew on every + * boot and the SEP has no run bit the AP can read, so only a + * resume from the image that another boot loaded can have gone + * through a boot; the PM core knows whether it did (a test + * resume does not). A hybrid sleep's wake is never taken for a + * cold boot. + */ + if (!pm_hibernate_is_restored()) { + event(SEP_PM_HIBERNATE_WARM); + return NOTIFY_OK; + } + cold = pm_hibernate_restored_cold(); + if (cold < 0) { + /* + * Cold or warm cannot be told. Sending to a freshly booted + * enclave, or registering again with a live one, are both + * worse than losing Touch ID until a reboot. + */ + if (dev) + dev_err(dev, "hibernation: no record of the boot that loaded the image, cannot tell a cold boot; the SEP session is dropped until a reboot\n"); + event(SEP_PM_HIBERNATE_COLD); + return NOTIFY_OK; + } + if (cold) { + if (event(SEP_PM_HIBERNATE_COLD) > 0 && dev) + sep_pm_schedule_reprobe(dev); + return NOTIFY_OK; + } + event(SEP_PM_HIBERNATE_WARM); return NOTIFY_OK; default: return NOTIFY_DONE; @@ -474,11 +572,12 @@ const struct dev_pm_ops sep_pm_ops = { .suspend_noirq = pm_sleep_ptr(sep_pm_suspend_noirq), }; -int sep_pm_register(void *ptr, void (*event)(bool entering), int (*sleep)(void), +int sep_pm_register(void *ptr, int (*event)(int event), int (*sleep)(void), void (*wake)(void), bool (*told)(void)) { int ret; + WRITE_ONCE(sep_pm_dev, (struct device *)ptr); WRITE_ONCE(sep_pm_event, event); WRITE_ONCE(sep_pm_sleep, sleep); WRITE_ONCE(sep_pm_wake, wake); @@ -489,6 +588,7 @@ int sep_pm_register(void *ptr, void (*event)(bool entering), int (*sleep)(void), WRITE_ONCE(sep_pm_sleep, NULL); WRITE_ONCE(sep_pm_wake, NULL); WRITE_ONCE(sep_pm_told, NULL); + WRITE_ONCE(sep_pm_dev, NULL); return ret; } @@ -503,6 +603,7 @@ void sep_pm_unregister(void) WRITE_ONCE(sep_pm_sleep, NULL); WRITE_ONCE(sep_pm_wake, NULL); WRITE_ONCE(sep_pm_told, NULL); + WRITE_ONCE(sep_pm_dev, NULL); } static void sep_pm_release_domains(void *data) diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 8c518ef4349e9..8c3d6d2f313d9 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -48,6 +48,12 @@ struct apple_rtkit_helper { bool idle_pending; /* Ends a sleep whose .prepare failed, which gets no .complete. */ struct notifier_block pm_nb; + /* Boot kernel of a resume from disk: RX held for the image. */ + bool rx_held; + /* This restore took over the session the boot kernel started. */ + bool readopted; + /* This restore found the co-processor of our own session halted. */ + bool halted_underneath; void __iomem *asc_base; @@ -63,6 +69,22 @@ bool apple_rtkit_helper_preserves_input(struct device *dev) } EXPORT_SYMBOL_GPL(apple_rtkit_helper_preserves_input); +bool apple_rtkit_helper_readopted(struct device *dev) +{ + struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + + return helper && helper->readopted; +} +EXPORT_SYMBOL_GPL(apple_rtkit_helper_readopted); + +bool apple_rtkit_helper_halted_underneath(struct device *dev) +{ + struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + + return helper && helper->halted_underneath; +} +EXPORT_SYMBOL_GPL(apple_rtkit_helper_halted_underneath); + static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) { struct apple_rtkit_helper *helper = cookie; @@ -119,9 +141,19 @@ static const struct apple_rtkit_ops apple_rtkit_helper_ops = { .shmem_destroy = apple_rtkit_helper_shmem_destroy, }; -static void apple_rtkit_helper_release(void *rtk) +/* Whether the coprocessor's CPU runs: halted, nothing answers on its mailbox. */ +static bool apple_rtkit_helper_cpu_running(struct apple_rtkit_helper *helper) +{ + return readl_relaxed(helper->asc_base + APPLE_ASC_CPU_CONTROL) & + APPLE_ASC_CPU_CONTROL_RUN; +} + +static void apple_rtkit_helper_release(void *data) { - apple_rtkit_free_retaining_buffers(rtk); + struct apple_rtkit_helper *helper = data; + + /* The current session object: a hibernation restore may have replaced it. */ + apple_rtkit_free_retaining_buffers(helper->rtk); } /* @@ -147,8 +179,7 @@ static int apple_rtkit_helper_live(struct apple_rtkit_helper *helper) return -EINVAL; } - if (!(readl_relaxed(helper->asc_base + APPLE_ASC_CPU_CONTROL) & - APPLE_ASC_CPU_CONTROL_RUN)) { + if (!apple_rtkit_helper_cpu_running(helper)) { dev_warn(helper->dev, "RTKit handoff without a running CPU\n"); return -ENODEV; } @@ -246,6 +277,21 @@ static int apple_rtkit_helper_inherit_buffers( return count; } +/* + * Whether an adopted session refuses buffers its inventory does not list. With + * an inventory, and on T8132 and the J614s without one, no new allocation is + * safe: the firmware still uses what it negotiated with its first host. + * Producers that predate the inventory (T8140) keep mapping IOVAs inside the + * SRAM region. Probe and the restore after a hibernation adopt by this rule. + */ +static bool apple_rtkit_helper_strict(struct device *dev, int count) +{ + return count > 0 || + of_device_is_compatible(dev->of_node, "apple,t8132-mtp") || + (of_machine_is_compatible("apple,j614s") && + of_device_is_compatible(dev->of_node, "apple,t6041-mtp")); +} + static int apple_rtkit_helper_pm_register(struct apple_rtkit_helper *helper); static int apple_rtkit_helper_probe(struct platform_device *pdev) @@ -330,16 +376,9 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) } if (live) { - /* - * Set before RX starts: with an inventory, or on T8132, no new - * buffer allocation is safe here. Producers that predate the - * inventory (T8140) keep mapping IOVAs inside the SRAM region. - */ + /* Set before RX starts, which may bring a buffer request. */ helper->adopted = true; - helper->strict_buffers = count > 0 || - of_device_is_compatible(dev->of_node, "apple,t8132-mtp") || - (of_machine_is_compatible("apple,j614s") && - of_device_is_compatible(dev->of_node, "apple,t6041-mtp")); + helper->strict_buffers = apple_rtkit_helper_strict(dev, count); helper->rtk = apple_rtkit_init_adopted_with_buffers( dev, helper, NULL, 0, &apple_rtkit_helper_ops, buffers, count); if (IS_ERR(helper->rtk)) @@ -347,7 +386,7 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) "Failed to adopt RTKit\n"); ret = devm_add_action_or_reset(dev, apple_rtkit_helper_release, - helper->rtk); + helper); if (ret) return ret; ret = apple_rtkit_helper_pm_register(helper); @@ -388,6 +427,17 @@ static void apple_rtkit_helper_remove(struct platform_device *pdev) if (helper->adopted || helper->preserved) return; + /* + * After a resume from disk the boot loader left the coprocessor halted + * with fresh firmware loaded, while the restored host session still + * believes it runs: nothing would answer the quiesce. Drop the session + * without a handshake so that the next probe starts the CPU again. + */ + if (!apple_rtkit_helper_cpu_running(helper)) { + dev_info(&pdev->dev, "co-processor halted underneath (resume from disk): removed without quiesce\n"); + return; + } + if (apple_rtkit_is_running(helper->rtk)) apple_rtkit_quiesce(helper->rtk); @@ -554,12 +604,194 @@ static int __maybe_unused apple_rtkit_helper_resume(struct device *dev) return 0; } -static int __maybe_unused apple_rtkit_helper_freeze(struct device *dev) +/* + * What the boot kernel of a resume from disk leaves for the restored kernel + * about a session it started itself: the shared buffers the firmware holds. + */ +struct apple_rtkit_helper_hib { + u32 count; + u32 reserved; + struct apple_rtkit_buffer_desc buf[APPLE_RTKIT_BUF_COUNT]; +}; +static_assert(sizeof(struct apple_rtkit_helper_hib) <= APPLE_RTKIT_HIB_HANDOFF_MAX); + +static void apple_rtkit_helper_hib_tag(struct device *dev, char *tag, size_t len) +{ + snprintf(tag, len, "rtkh:%s", dev_name(dev)); +} + +/* + * Hibernation of an adopted session. The image is written while the firmware + * keeps running; its buffers live in SRAM or in no-map reserved RAM, so they + * are not part of the image and the host side has nothing to quiesce. The + * restore re-adopts the instance the boot loader started for the resume boot, + * exactly as probe adopted the first one. + * + * A session this driver started itself is left running as well: the MTP + * firmware does not come back from QUIESCED (it does not answer the endpoint + * discovery of the next start). When this is the boot kernel of a resume from + * disk (it started MTP so that the disk passphrase can be typed), it hands + * the running firmware to the image: mailbox RX is held, so that what the + * firmware sends now stays queued for the restored kernel, the messages + * already received are handled, and the shared buffers are described for the + * restored kernel, which can only take the session over if its own buffers + * sit at the same IOVAs (the firmware keeps writing to the IOVAs it was + * given, and after the restore the DART translates them through the image's + * page tables). + */ +static int apple_rtkit_helper_freeze(struct device *dev) { struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + struct apple_rtkit_helper_hib rec = {}; + char tag[32]; + int n, ret; + + helper->readopted = false; + helper->halted_underneath = false; + if (helper->adopted || !apple_rtkit_hib_restoring()) + return 0; + if (!apple_rtkit_helper_cpu_running(helper) || + !apple_rtkit_is_running(helper->rtk)) + return 0; + + apple_rtkit_hold_rx(helper->rtk); + apple_rtkit_flush_rx(helper->rtk); + helper->rx_held = true; - /* A hibernation image cannot recreate the firmware's live DMA ownership. */ - return helper->adopted ? -EBUSY : 0; + n = apple_rtkit_describe_buffers(helper->rtk, rec.buf, ARRAY_SIZE(rec.buf)); + if (n < 0) { + dev_warn(dev, "resume from disk: cannot describe the RTKit buffers: %d\n", n); + return 0; + } + rec.count = n; + apple_rtkit_helper_hib_tag(dev, tag, sizeof(tag)); + ret = apple_rtkit_hib_handoff_put(tag, &rec, sizeof(rec)); + if (ret) + dev_warn(dev, "resume from disk: no hand-off record for the image: %d\n", ret); + else + dev_dbg(dev, "resume from disk: RTKit session left running for the image (%d buffers)\n", + n); + return 0; +} + +static int apple_rtkit_helper_thaw(struct device *dev) +{ + struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + char tag[32]; + + /* The jump into the image failed: this kernel keeps the session. */ + if (helper->rx_held) { + helper->rx_held = false; + apple_rtkit_helper_hib_tag(dev, tag, sizeof(tag)); + apple_rtkit_hib_handoff_drop(tag); + apple_rtkit_resume_rx(helper->rtk); + } + return 0; +} + +/* + * A session this driver started itself, after a resume from disk. Taken over + * in place when the boot kernel left it running and its buffers match ours; + * otherwise left as before (stale until the driver is reloaded). + */ +static int apple_rtkit_helper_restore_own(struct apple_rtkit_helper *helper) +{ + struct device *dev = helper->dev; + struct apple_rtkit_helper_hib rec = {}, own = {}; + char tag[32]; + int i, n, ret; + + apple_rtkit_helper_hib_tag(dev, tag, sizeof(tag)); + ret = apple_rtkit_hib_handoff_take(tag, &rec, sizeof(rec)); + if (ret < 0) { + if (apple_rtkit_helper_cpu_running(helper)) { + dev_warn(dev, "resume from disk: co-processor runs without a hand-off from the boot kernel (%d): not taken over\n", + ret); + } else { + helper->halted_underneath = true; + dev_info(dev, "resume from disk: self-started RTKit session left for a reload\n"); + } + return 0; + } + if (!apple_rtkit_helper_cpu_running(helper)) { + helper->halted_underneath = true; + dev_warn(dev, "resume from disk: hand-off present but the co-processor is halted: not taken over\n"); + return 0; + } + + n = apple_rtkit_describe_buffers(helper->rtk, own.buf, ARRAY_SIZE(own.buf)); + own.count = n < 0 ? 0 : n; + if (n < 0 || rec.count > ARRAY_SIZE(rec.buf) || own.count != rec.count || + memcmp(own.buf, rec.buf, rec.count * sizeof(rec.buf[0]))) { + dev_err(dev, "resume from disk: RTKit buffers of the boot kernel differ from the image's: not taken over\n"); + for (i = 0; i < min_t(u32, rec.count, ARRAY_SIZE(rec.buf)); i++) + dev_err(dev, " boot kind %u iova %#llx size %#llx\n", + rec.buf[i].kind, rec.buf[i].iova, rec.buf[i].size); + for (i = 0; i < own.count; i++) + dev_err(dev, " image kind %u iova %#llx size %#llx\n", + own.buf[i].kind, own.buf[i].iova, own.buf[i].size); + return 0; + } + + ret = apple_rtkit_readopt(helper->rtk); + if (ret) { + dev_err(dev, "resume from disk: RTKit session not taken over: %d\n", ret); + return 0; + } + helper->readopted = true; + dev_dbg(dev, "resume from disk: took over the RTKit session the boot kernel started (%d buffers at the same IOVAs)\n", + n); + return 0; +} + +static int apple_rtkit_helper_restore(struct device *dev) +{ + struct apple_rtkit_helper *helper = dev_get_drvdata(dev); + struct apple_rtkit_inherited_buffer buffers[APPLE_RTKIT_HELPER_MAX_BUFFERS] = {}; + struct apple_rtkit *old, *rtk; + int count, live; + + if (!helper->adopted) + return apple_rtkit_helper_restore_own(helper); + + /* + * The restored tree still carries the handoff the image's boot stage + * wrote. The resume boot's loader hands the same firmware over at the + * same SRAM and no-map addresses (the inventory comes from the firmware + * carveouts, not from the kernel allocator), so re-read it and check + * that a CPU is running behind it. + */ + live = apple_rtkit_helper_live(helper); + if (live < 0) { + dev_err(dev, "hibernation restore: invalid live RTKit handoff: %d\n", live); + return live; + } + if (!live) { + dev_err(dev, "hibernation restore: no live RTKit session to re-adopt\n"); + return -ENODEV; + } + count = apple_rtkit_helper_inherit_buffers(helper, buffers); + if (count < 0) { + dev_err(dev, "hibernation restore: invalid inherited RTKit buffers: %d\n", count); + return count; + } + + /* As at probe, before the new session's RX starts. */ + helper->strict_buffers = apple_rtkit_helper_strict(dev, count); + rtk = apple_rtkit_init_adopted_with_buffers(dev, helper, NULL, 0, + &apple_rtkit_helper_ops, buffers, count); + if (IS_ERR(rtk)) { + dev_err(dev, "hibernation restore: failed to re-adopt RTKit: %pe\n", rtk); + return PTR_ERR(rtk); + } + + /* The old object described an instance that no longer exists. */ + old = helper->rtk; + helper->rtk = rtk; + apple_rtkit_free_retaining_buffers(old); + dev_dbg(dev, "re-adopted the running RTKit session after hibernation (%d retained buffers)\n", + count); + return 0; } static const struct dev_pm_ops apple_rtkit_helper_pm_ops = { @@ -569,6 +801,8 @@ static const struct dev_pm_ops apple_rtkit_helper_pm_ops = { .resume = pm_sleep_ptr(apple_rtkit_helper_resume), .freeze = pm_sleep_ptr(apple_rtkit_helper_freeze), .poweroff = pm_sleep_ptr(apple_rtkit_helper_freeze), + .thaw = pm_sleep_ptr(apple_rtkit_helper_thaw), + .restore = pm_sleep_ptr(apple_rtkit_helper_restore), }; static const struct of_device_id apple_rtkit_helper_of_match[] = { diff --git a/drivers/soc/apple/rtkit-hib-handoff.c b/drivers/soc/apple/rtkit-hib-handoff.c new file mode 100644 index 0000000000000..0700ab8488284 --- /dev/null +++ b/drivers/soc/apple/rtkit-hib-handoff.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple RTKit: hand a running co-processor session from the kernel that loads + * a hibernation image (the boot kernel) to the kernel restored from it. + * + * A co-processor that the boot kernel started keeps running across the jump + * into the image, but everything the boot kernel knew about it is overwritten + * by the image. The few facts the restored kernel needs to take the session + * over (where the firmware's shared buffers are, how far a receive ring got) + * are left in __nosavedata: the image never saves those pages, so after the + * restore they still hold what the boot kernel wrote, provided both kernels + * are the same build loaded at the same physical address. Each record carries + * its own physical address and a checksum, so a record from a kernel loaded + * elsewhere, or no record at all, reads as absent. + * + * The storage must be in the kernel image, so this only works with RTKit + * built in. A taken record is cleared. + */ + +#include +#include +#include +#include +#include + +#if IS_BUILTIN(CONFIG_APPLE_RTKIT) && IS_ENABLED(CONFIG_HIBERNATION) + +#define HANDOFF_MAGIC 0x52544b48 /* "RTKH" */ +#define HANDOFF_VERSION 1 +#define HANDOFF_SLOTS 4 +#define HANDOFF_TAG 32 + +struct apple_rtkit_hib_slot { + u32 magic; + u32 version; + u64 self_phys; + char tag[HANDOFF_TAG]; + u32 len; + u32 sum; + u8 data[APPLE_RTKIT_HIB_HANDOFF_MAX]; +}; + +static struct apple_rtkit_hib_slot handoff[HANDOFF_SLOTS] __nosavedata; +static DEFINE_SPINLOCK(handoff_lock); + +/* FNV-1a over everything but the checksum itself. */ +static u32 handoff_sum(const struct apple_rtkit_hib_slot *s) +{ + const u8 *p = (const u8 *)s; + size_t off = offsetof(struct apple_rtkit_hib_slot, sum); + u32 h = 0x811c9dc5; + size_t i; + + for (i = 0; i < sizeof(*s); i++) { + if (i >= off && i < off + sizeof(s->sum)) + continue; + h ^= p[i]; + h *= 0x01000193; + } + return h; +} + +static bool handoff_valid(const struct apple_rtkit_hib_slot *s) +{ + return s->magic == HANDOFF_MAGIC && s->version == HANDOFF_VERSION && + s->self_phys == __pa_symbol(s) && + s->len <= APPLE_RTKIT_HIB_HANDOFF_MAX && + s->tag[HANDOFF_TAG - 1] == '\0' && s->sum == handoff_sum(s); +} + +static struct apple_rtkit_hib_slot *handoff_find(const char *tag) +{ + int i; + + for (i = 0; i < HANDOFF_SLOTS; i++) + if (handoff_valid(&handoff[i]) && !strcmp(handoff[i].tag, tag)) + return &handoff[i]; + return NULL; +} + +bool apple_rtkit_hib_restoring(void) +{ + return pm_hibernate_is_loading(); +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_restoring); + +int apple_rtkit_hib_handoff_put(const char *tag, const void *data, size_t len) +{ + struct apple_rtkit_hib_slot *s; + unsigned long flags; + int i, ret = 0; + + if (len > APPLE_RTKIT_HIB_HANDOFF_MAX || strlen(tag) >= HANDOFF_TAG) + return -EINVAL; + + spin_lock_irqsave(&handoff_lock, flags); + s = handoff_find(tag); + for (i = 0; !s && i < HANDOFF_SLOTS; i++) + if (!handoff_valid(&handoff[i])) + s = &handoff[i]; + if (s) { + memset(s, 0, sizeof(*s)); + s->magic = HANDOFF_MAGIC; + s->version = HANDOFF_VERSION; + s->self_phys = __pa_symbol(s); + strscpy(s->tag, tag, sizeof(s->tag)); + s->len = len; + memcpy(s->data, data, len); + s->sum = handoff_sum(s); + } else { + ret = -ENOSPC; + } + spin_unlock_irqrestore(&handoff_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_put); + +int apple_rtkit_hib_handoff_take(const char *tag, void *data, size_t len) +{ + struct apple_rtkit_hib_slot *s; + unsigned long flags; + int ret; + + spin_lock_irqsave(&handoff_lock, flags); + s = handoff_find(tag); + if (!s) { + ret = -ENOENT; + } else if (s->len != len) { + ret = -EINVAL; + } else { + memcpy(data, s->data, len); + ret = len; + } + if (s) + memset(s, 0, sizeof(*s)); + spin_unlock_irqrestore(&handoff_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_take); + +void apple_rtkit_hib_handoff_drop(const char *tag) +{ + struct apple_rtkit_hib_slot *s; + unsigned long flags; + + spin_lock_irqsave(&handoff_lock, flags); + s = handoff_find(tag); + if (s) + memset(s, 0, sizeof(*s)); + spin_unlock_irqrestore(&handoff_lock, flags); +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_drop); + +#else + +bool apple_rtkit_hib_restoring(void) +{ + return false; +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_restoring); + +int apple_rtkit_hib_handoff_put(const char *tag, const void *data, size_t len) +{ + return -EOPNOTSUPP; +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_put); + +int apple_rtkit_hib_handoff_take(const char *tag, void *data, size_t len) +{ + return -ENOENT; +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_take); + +void apple_rtkit_hib_handoff_drop(const char *tag) +{ +} +EXPORT_SYMBOL_GPL(apple_rtkit_hib_handoff_drop); + +#endif diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 8230ebe11d411..1f7b101d5d195 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1641,6 +1641,63 @@ bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep) } EXPORT_SYMBOL_GPL(apple_rtkit_has_endpoint); +int apple_rtkit_describe_buffers(struct apple_rtkit *rtk, + struct apple_rtkit_buffer_desc *desc, + unsigned int max) +{ + const struct apple_rtkit_shmem *bufs[APPLE_RTKIT_BUF_COUNT] = { + [APPLE_RTKIT_BUF_IOREPORT] = &rtk->ioreport_buffer, + [APPLE_RTKIT_BUF_COREANALYTICS] = &rtk->coreanalytics_buffer, + [APPLE_RTKIT_BUF_CRASHLOG] = &rtk->crashlog_buffer, + [APPLE_RTKIT_BUF_OSLOG] = &rtk->oslog_buffer, + [APPLE_RTKIT_BUF_KDEBUG0] = &rtk->kdebug_buffer0, + [APPLE_RTKIT_BUF_KDEBUG1] = &rtk->kdebug_buffer1, + [APPLE_RTKIT_BUF_PMP_AUX] = &rtk->pmp_aux_buffer, + [APPLE_RTKIT_BUF_SYSLOG] = &rtk->syslog_buffer, + }; + unsigned int i, n = 0; + + for (i = 0; i < APPLE_RTKIT_BUF_COUNT; i++) { + if (!bufs[i]->size) + continue; + if (n == max) + return -ENOSPC; + desc[n].kind = i; + desc[n].is_mapped = bufs[i]->is_mapped; + desc[n].iova = bufs[i]->iova; + desc[n].size = bufs[i]->size; + n++; + } + return n; +} +EXPORT_SYMBOL_GPL(apple_rtkit_describe_buffers); + +int apple_rtkit_readopt(struct apple_rtkit *rtk) +{ + /* + * Only a session that was running when it was left can be taken over: + * its endpoint map, protocol version, syslog geometry and buffers then + * describe the same firmware build and the same IOVAs as the instance + * that runs now (the caller compared the buffers). + */ + if (READ_ONCE(rtk->shutting_down) || !apple_rtkit_is_running(rtk)) + return -EINVAL; + + apple_mbox_hold_rx(rtk->mbox); + /* Acknowledgments still queued belong to the instance that is gone. */ + cancel_work_sync(&rtk->sync_ack_work); + flush_workqueue(rtk->wq); + kfifo_reset(&rtk->sync_ack_fifo); + + reinit_completion(&rtk->iop_pwr_ack_completion); + reinit_completion(&rtk->ap_pwr_ack_completion); + + /* Messages the other kernel left queued in the mailbox arrive now. */ + apple_mbox_release_rx(rtk->mbox); + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_readopt); + int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) { u64 msg; @@ -2146,8 +2203,9 @@ static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk, return 0; } -static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, - unsigned int state) +/* Ask the IOP for a power state and wait for its acknowledgment; the caller judges the state. */ +static int apple_rtkit_send_iop_power_state(struct apple_rtkit *rtk, + unsigned int state) { u64 msg; int ret; @@ -2160,7 +2218,15 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, if (ret) return ret; - ret = apple_rtkit_wait(rtk, &rtk->iop_pwr_ack_completion); + return apple_rtkit_wait(rtk, &rtk->iop_pwr_ack_completion); +} + +static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, + unsigned int state) +{ + int ret; + + ret = apple_rtkit_send_iop_power_state(rtk, state); if (ret) return ret; @@ -2298,11 +2364,16 @@ int apple_rtkit_idle_with(struct apple_rtkit *rtk, void (*after_ap)(void *data), if (after_ap) after_ap(data); - ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); + ret = apple_rtkit_send_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); if (ret) { dev_err(rtk->dev, "RTKit: IOP idle not acknowledged: %d\n", ret); return ret; } + if (rtk->iop_power_state != APPLE_RTKIT_PWR_STATE_IDLE) { + dev_dbg(rtk->dev, "RTKit: IOP acknowledged %#x instead of idle\n", + rtk->iop_power_state); + return APPLE_RTKIT_IDLE_RETAINED; + } rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE; @@ -2514,18 +2585,21 @@ static int apple_rtkit_polled_state(struct apple_rtkit *rtk, u8 type, unsigned i return -ETIMEDOUT; } -int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, - unsigned int *us) +/* The receive path while a notice is polled for, and the usual one after it. */ +static void apple_rtkit_polled_set_rx(struct apple_rtkit *rtk, + void (*rx)(struct apple_mbox *mbox, + struct apple_mbox_msg msg, void *cookie)) { unsigned long flags; - u64 owed; - u32 status; - int ret; - *us = 0; spin_lock_irqsave(&rtk->mbox->rx_lock, flags); - rtk->mbox->rx = apple_rtkit_polled_rx; + rtk->mbox->rx = rx; spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); +} + +static void apple_rtkit_polled_owed(struct apple_rtkit *rtk) +{ + u64 owed; /* What's already waiting is answered first, or the firmware may not go idle. */ apple_mbox_poll(rtk->mbox); @@ -2537,6 +2611,17 @@ int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_con if (!apple_rtkit_polled_send(rtk, echo)) dev_err(rtk->dev, "RTKit: oslog echo failed before the idle\n"); } +} + +int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_control, + unsigned int *us) +{ + u32 status; + int ret; + + *us = 0; + apple_rtkit_polled_set_rx(rtk, apple_rtkit_polled_rx); + apple_rtkit_polled_owed(rtk); ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, &rtk->ap_power_state, us); @@ -2549,14 +2634,43 @@ int apple_rtkit_sleep_idle_polled(struct apple_rtkit *rtk, void __iomem *cpu_con APPLE_ASC_CPU_STATUS_DOWN, APPLE_RTKIT_POLLED_STEP_US, 5000); - spin_lock_irqsave(&rtk->mbox->rx_lock, flags); - rtk->mbox->rx = apple_rtkit_rx; - spin_unlock_irqrestore(&rtk->mbox->rx_lock, flags); + apple_rtkit_polled_set_rx(rtk, apple_rtkit_rx); return ret; } EXPORT_SYMBOL_GPL(apple_rtkit_sleep_idle_polled); +/* + * For a co-processor that stays up: its client may still be owed a reply or an event on + * its own endpoints. Those go the usual way, to recv_message_early and, if it leaves + * them, to the receive work, which runs again after the wake. + */ +static void apple_rtkit_ap_polled_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, + void *cookie) +{ + if (msg.msg1 >= APPLE_RTKIT_APP_ENDPOINT_START) + apple_rtkit_rx(mbox, msg, cookie); + else + apple_rtkit_polled_rx(mbox, msg, cookie); +} + +int apple_rtkit_ap_idle_polled(struct apple_rtkit *rtk, unsigned int *us) +{ + int ret; + + *us = 0; + apple_rtkit_polled_set_rx(rtk, apple_rtkit_ap_polled_rx); + apple_rtkit_polled_owed(rtk); + + ret = apple_rtkit_polled_state(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + &rtk->ap_power_state, us); + + apple_rtkit_polled_set_rx(rtk, apple_rtkit_rx); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_ap_idle_polled); + void apple_rtkit_mailbox_recv_off(struct apple_rtkit *rtk) { apple_mbox_set_recv_enabled(rtk->mbox, false); diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 938e9b9a529a7..f32945e01f5df 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -26,7 +26,14 @@ void sep_cancel_delayed_work_sync(void *work); /* -- pm_shim.c ---------------------------------------------------------- */ -int sep_pm_register(void *dev, void (*event)(bool entering), int (*sleep)(void), +/* Sleep events for sep_pm_register()'s callback; mirrored in pm.rs. */ +#define SEP_PM_SLEEP 0 +#define SEP_PM_WAKE 1 +#define SEP_PM_HIBERNATE 2 +#define SEP_PM_HIBERNATE_WARM 3 +#define SEP_PM_HIBERNATE_COLD 4 +#define SEP_PM_RESTORE 5 +int sep_pm_register(void *dev, int (*event)(int event), int (*sleep)(void), void (*wake)(void), bool (*told)(void)); void sep_pm_unregister(void); int sep_pm_keep_domains(void *dev); diff --git a/drivers/soc/apple/sleep-internal.h b/drivers/soc/apple/sleep-internal.h new file mode 100644 index 0000000000000..c5286dbcb088e --- /dev/null +++ b/drivers/soc/apple/sleep-internal.h @@ -0,0 +1,332 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * The coprocessor and enclave registry, and what the Apple SoC sleep driver + * (sleep.c) shares with a SoC's own part of it (sleep-t6040.c for T6040). + * Drivers register through ; the controller bound on + * this SoC walks the table at its commit. + */ +#ifndef _APPLE_SLEEP_INTERNAL_H_ +#define _APPLE_SLEEP_INTERNAL_H_ + +#include +#include +#include +#include +#include +#include + +#include + +struct device; +struct device_node; +struct platform_device; +struct seq_file; + +/* The coprocessors the last mile stops. Each driver idles its own first. */ +#define APPLE_SLEEP_COPROCS 6 + +struct apple_sleep_coproc { + struct device *dev; + void __iomem *asc; + bool pcie_ans; + int (*last_check)(void *data); + int (*last_stop)(void *data); + void *data; + bool ready; + bool stopped; + u32 status; + int stop_ret; + /* No ASC: its last stop takes it whole, power word @ps included, before the others. */ + bool first; + unsigned int order; /* among those registered first, lowest runs first */ + u32 ps; +}; + +/* Irreversible enclave stop; nonzero is legal only if SLEEP was not sent. */ +struct apple_sleep_enclave { + struct device *dev; + int (*stop)(void *data); + void *data; +}; + +extern struct apple_sleep_coproc apple_sleep_coprocs[APPLE_SLEEP_COPROCS]; +extern struct apple_sleep_enclave apple_sleep_enclave; +extern struct mutex apple_sleep_coprocs_lock; + +struct apple_sleep_coproc *apple_sleep_coproc_find(struct device *dev); + +/* + * The sleep controller bound on this SoC. Each binds only to its own SoC's + * compatible, so at most one is bound. Bound, it tells the registry when its + * probe is complete: from then on a suspend through firmware that is not a dry + * run is one whose commit stops the registered coprocessors. + * + * APPLE_SLEEP_REG_ENCLAVE_ANY: take every enclave registration. Without it one + * is taken only while such a commit is the case. + * APPLE_SLEEP_REG_LID_AT_WAKE, APPLE_SLEEP_REG_USB_WAKE_PORTS: the sleep takes the lid's state + * at the wake, and the USB wake ports. Without them + * apple_sleep_lid_closed_at_wake() and apple_sleep_usb_port_can_wake() answer as they do + * with no sleep driver, whatever the SMC's and the USB host's drivers told. + */ +#define APPLE_SLEEP_REG_ENCLAVE_ANY BIT(0) +#define APPLE_SLEEP_REG_LID_AT_WAKE BIT(1) +#define APPLE_SLEEP_REG_USB_WAKE_PORTS BIT(2) + +int apple_sleep_registry_bind(unsigned int flags); +void apple_sleep_registry_unbind(void); +void apple_sleep_registry_ready(void); + +/* Must match aura's struct aura_s2r_mailbox. */ +#define AURA_S2R_MAGIC 0x3152325341525541ULL /* "AURAS2R1" */ + +struct aura_s2r_mailbox { + u64 magic; /* aura, before the kernel starts */ + u64 boot_mpidr; /* aura: the only core that may take resume_pa */ + u64 resume_pa; /* kernel, armed before a commit; aura clears it */ + u64 resumes; /* aura: resume_pa taken this many times */ +}; + +/* + * Registers that are the same on every SoC this driver knows. + */ + +/* PS word */ +#define PS_AUTO_ENABLE BIT(28) +#define PS_MIN GENMASK(19, 16) +#define PS_UNLATCH BIT(12) +#define PS_BUSY BIT(11) +#define PS_DEV_DISABLE BIT(10) +#define PS_FLAGS GENMASK(9, 8) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_TARGET GENMASK(3, 0) +#define PS_ACTIVE 0xf +#define PS_OFF_TIMEOUT_US 5000 +#define PS_BUSY_SETTLE_US 1000 + +/* CPU start block, as aura's smp_start_cpu() uses it. A core's bit is the SoC's. */ +#define CPU_START_STOP 0x0 +#define CPU_START_SYS 0x4 +#define CPU_START_CLUSTER(n) (0x8 + (n) * 4) + +/* Watchdog block, ADT /arm-io/wdt reg[0]. WD2 is the sleep safety net. */ +#define WDT_WD0_COUNT 0x00 +#define WDT_WD0_CTL 0x0c +#define WDT_WD1_COUNT 0x10 +#define WDT_WD2_COUNT 0x20 +#define WDT_WD2_ALARM 0x28 +#define WDT_WD2_CTL 0x2c +#define WDT_TICKS_PER_S 24000000 +/* The sleep's own safety net: ADT /arm-io/wdt simple-reconfig-wdog-icc-time. */ +#define WDT_SLEEP_SECS 5 +/* Device suspend and resume can take a while, and nothing else recovers a hang in them. */ +#define WDT_SUSPEND_SECS 120 + +/* The AP queue of the PMU's SPMI controller, and its response vector's enable and ack */ +#define SPMI_STATUS 0x00 +#define SPMI_STATUS_RX_EMPTY BIT(24) +#define SPMI_CMD 0x04 +#define SPMI_REPLY 0x08 +#define SPMI_NOTIFY_ENABLE 0x40 +#define SPMI_NOTIFY_REPLY BIT(0) +#define SPMI_NOTIFY_ACK 0x80 +#define SPMI_CMD_ACTIVE BIT(15) +#define SPMI_OP_EXT_WRITE(n) ((n) - 1) +#define SPMI_OP_EXT_READ(n) (0x20 | ((n) - 1)) +#define SPMI_OP_ZERO_WRITE 0x80 +#define SPMI_OP_SLEEP 0x11 +#define SPMI_OP_WAKEUP 0x13 +#define SPMI_POLL_US 20000 + +/* A coprocessor's ASC */ +#define ASC_CPU_CONTROL 0x44 +#define ASC_CPU_CONTROL_RUN BIT(4) +#define ASC_CPU_STATUS 0x48 + +/* The part of the PCIe block that ANS shares, whether or not its host is bound. */ +#define APCIE_AXI_EN0 0x104 +#define APCIE_AXI_EN1 0x108 +#define APCIE_CLK_CTRL 0x0 +#define APCIE_CLK_BUSY BIT(31) +#define APCIE_CLK_RUN BIT(5) +#define APCIE_CLK_OFF BIT(1) +#define APCIE_CLK_OFF2 BIT(2) +#define APCIE_CLK_START 0x4 +#define APCIE_CLK_START_GO BIT(31) +#define APCIE_PHY_COMMON 0x4000 +#define APCIE_PHY_CTRL 0x8000 +#define APCIE_PHY_CTRL_CLK0REQ BIT(0) +#define APCIE_PHY_CTRL_CLK1REQ BIT(1) +#define APCIE_PHY_CTRL_CLK0ACK BIT(2) +#define APCIE_PHY_CTRL_CLK1ACK BIT(3) +#define APCIE_PHY_CTRL_RESET BIT(4) +#define APCIE_PHY_CTRL_UP BIT(27) +#define APCIE_PHY_RUN 0x8004 +#define APCIE_PHY_RUN_EN BIT(0) +#define APCIE_PHY_RUN_EN2 BIT(4) +#define APCIE_PHY_READY 0x8008 +#define APCIE_PHY_IP_PLL_CTL 0x90 +#define APCIE_PHY_IP_PLL_HOLD BIT(16) +#define APCIE_PORT_INTSTAT 0x100 +#define APCIE_PORT_INTMSK 0x104 +#define APCIE_PORT_INT_LINK GENMASK(7, 4) +#define APCIE_PORT_L2 0x88 +#define APCIE_PORT_L2_ENTER 0x11 +#define APCIE_PORT_L2_BUSY BIT(0) +#define APCIE_PORT_PERST_GUARD 0x13c +#define APCIE_PORT_LINKSTS 0x208 +#define APCIE_PORT_LINKSTS_UP BIT(0) +#define APCIE_PORT_APPCLK 0x800 +#define APCIE_PORT_APPCLK_EN BIT(0) +#define APCIE_PORT_APPCLK_CGDIS BIT(8) +#define APCIE_PORT_STATUS 0x804 +#define APCIE_PORT_STATUS_READY BIT(0) +#define APCIE_PORT_PERST 0x82c +#define APCIE_PORT_PHY_LINK BIT(31) +#define APCIE_PORT_PHY_EN0 BIT(9) +#define APCIE_PORT_PHY_EN1 BIT(10) +#define APCIE_ENGINE_CTRL 0x0 +#define APCIE_ENGINE_IRQ_MASK 0x8 +#define APCIE_ENGINE_STOP BIT(0) +#define APCIE_FABRIC_STOP 0x4 +#define APCIE_PORT_STOP_US 5000 +#define APCIE_RC_ZERO 0x04 +#define APCIE_RC_READY_START 0x50 +#define APCIE_RC_READY_CFG 0x54 +#define APCIE_RC_READY_STAT 0x58 +#define APCIE_POLL_US 50000 + +/* + * Written with bits 0 and 63 by every core right before its last WFI. This is not + * CYC_OVRD, which is undefined on this SoC. + */ +#define SYS_APL_DEEP_SLEEP sys_reg(3, 5, 15, 6, 2) +#define DEEP_SLEEP_ENABLE (BIT_ULL(0) | BIT_ULL(63)) + +#define SYS_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) +#define SYS_APL_SIQ_CFG_EL1 sys_reg(3, 4, 15, 10, 4) + +/* A one-byte register write on the sleep's SPMI bus, to the PMU's USID */ +#define SPMI_CMD_WRITE (BIT(15) | 0x30) +#define SPMI_CMD_USID GENMASK(11, 8) + +/* A PMU write: the reply's vector masked, then acknowledged, so none is left raised. */ +static inline void apple_sleep_pmu_write(void __iomem *spmi, u8 usid, u16 reg, u8 val) +{ + u32 status, enable; + + enable = readl(spmi + SPMI_NOTIFY_ENABLE); + writel(enable & ~SPMI_NOTIFY_REPLY, spmi + SPMI_NOTIFY_ENABLE); + writel(((u32)reg << 16) | SPMI_CMD_WRITE | FIELD_PREP(SPMI_CMD_USID, usid), spmi + SPMI_CMD); + writel(val, spmi + SPMI_CMD); + if (!readl_poll_timeout_atomic(spmi + SPMI_STATUS, status, + !(status & SPMI_STATUS_RX_EMPTY), 1, 1000)) + readl(spmi + SPMI_REPLY); + writel(1, spmi + SPMI_NOTIFY_ACK); + writel(enable, spmi + SPMI_NOTIFY_ENABLE); +} + +/* + * A domain is raised with the target, then AUTO_ENABLE once it reads on. Measured + * byte for byte on ANS and PCIe, for instance 0x0f002300, 0x0f00200f, 0x1f00200f. + */ +static inline int apple_sleep_ps_raise_start(void __iomem *reg) +{ + u32 now; + + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET, reg); + + return readl_poll_timeout_atomic(reg, now, (now & PS_ACTUAL) == PS_ACTUAL, 1, + PS_OFF_TIMEOUT_US); +} + +static inline void apple_sleep_ps_raise_finish(void __iomem *reg) +{ + writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET | PS_AUTO_ENABLE, + reg); +} + +/* Read-modify-write in table order, with no write where nothing changes. */ +static inline void apple_sleep_tunable_apply_changed(void __iomem *regs, + const struct apple_tunable *t) +{ + size_t i; + + for (i = 0; i < t->sz; i++) { + void __iomem *reg = regs + t->values[i].offset; + u32 old = readl(reg); + u32 val = (old & ~t->values[i].mask) | (t->values[i].value & t->values[i].mask); + + if (val != old) + writel(val, reg); + } +} + +/* WD2 of the AON watchdog, the sleep's safety net, armed for @secs from now. */ +static inline void apple_sleep_wd2_arm(void __iomem *wdt, unsigned int secs) +{ + writel(0, wdt + WDT_WD2_CTL); + writel(0, wdt + WDT_WD2_COUNT); + writel(secs * WDT_TICKS_PER_S, wdt + WDT_WD2_ALARM); + writel(1, wdt + WDT_WD2_CTL); +} + +/* + * A SoC whose sleep doesn't fit the lists of struct apple_sleep_soc brings its own sequences. + * @probe then runs in place of the driver's own, and installs the part's suspend operations. + * @restore, if set, runs when the system comes back from a hibernation image. + * @registry_flags: how the part binds to the registry, which the driver does around @probe. + */ +struct apple_sleep_part { + int (*probe)(struct platform_device *pdev); + int (*restore)(struct device *dev); + unsigned int registry_flags; +}; + +/* sleep-t6040.c, built where the SMC driver is built in */ +#define APPLE_SLEEP_T6040 IS_BUILTIN(CONFIG_MFD_MACSMC) +extern const struct apple_sleep_part t6040_sleep_part; + +/* + * For a part with dry runs, from its .begin to its .end: 1 runs the firmware-driven device + * suspend short of what only a power cycle restores, 2 also the last steps that can still be + * taken back. apple_sleep_power_loss() and apple_sleep_last_steps() answer by it. + */ +void apple_sleep_set_dry(unsigned int level); + +/* + * The storage PCIe block's windows and tunables are on a node a loader fills. A tunable table + * @prop for the window @name, or NULL; and the window itself mapped, with its tunable if @prop + * names one, or NULL where either is missing. + */ +struct apple_tunable *apple_sleep_apcie_tunable(struct device *dev, struct device_node *np, + const char *name, const char *prop); +void __iomem *apple_sleep_apcie_block(struct device *dev, struct device_node *np, + const char *name, const char *prop, + struct apple_tunable **tunable); + +/* Each sleep begins with no answer about the wake before it. */ +void apple_sleep_wake_info_forget(void); + +/* For a debugfs file: who is registered, and how far each is. */ +void apple_sleep_registry_show(struct seq_file *s); + +/* + * The rest is for the commit point, in this order: interrupts off and one CPU up, so nothing + * takes the registry's lock. @dev is the sleep driver's, for the log. + * + * Every registered coprocessor has to be ready: returns how many aren't, each logged, and sets + * @pcie_ans_idled if ANS is among the ready ones. + */ +unsigned int apple_sleep_coprocs_refuse(struct device *dev, bool *pcie_ans_idled); +/* Each ready coprocessor's last check: the first refusal, or 0. */ +int apple_sleep_coprocs_check(struct device *dev); +bool apple_sleep_enclave_registered(void); +/* A power word that a ready coprocessor's last stop lowers: on until the commit. */ +bool apple_sleep_coproc_lowers(u32 ps); +/* The enclave's stop, last of all refusals: nonzero only if no request was sent. */ +int apple_sleep_enclave_stop(struct device *dev); +/* The last stops, those registered first with @first and then the rest. */ +void apple_sleep_coprocs_stop(bool first, bool *pcie_ans); + +#endif diff --git a/drivers/soc/apple/sleep-registry.c b/drivers/soc/apple/sleep-registry.c new file mode 100644 index 0000000000000..5903c0fbef337 --- /dev/null +++ b/drivers/soc/apple/sleep-registry.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC sleep: the coprocessor and enclave registry + * + * A driver whose coprocessor can only be stopped once a sleep is certain + * registers it here, readies it in its own suspend and leaves the last stop to + * the commit of whichever sleep controller is bound on this SoC (sleep.c, + * sleep-t6040.c). The table lives apart from the controllers so that one + * kernel can carry all of them; each binds only to its own SoC. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "sleep-internal.h" + +struct apple_sleep_coproc apple_sleep_coprocs[APPLE_SLEEP_COPROCS]; +struct apple_sleep_enclave apple_sleep_enclave; +/* Guards the table, the enclave slot and the bound controller. */ +DEFINE_MUTEX(apple_sleep_coprocs_lock); + +#define APPLE_SLEEP_REG_BOUND BIT(8) +#define APPLE_SLEEP_REG_READY BIT(9) + +/* The bound controller's flags and state. */ +static unsigned int apple_sleep_reg; + +int apple_sleep_registry_bind(unsigned int flags) +{ + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_reg & APPLE_SLEEP_REG_BOUND) + return -EBUSY; + WRITE_ONCE(apple_sleep_reg, flags | APPLE_SLEEP_REG_BOUND); + + return 0; +} + +void apple_sleep_registry_unbind(void) +{ + guard(mutex)(&apple_sleep_coprocs_lock); + WRITE_ONCE(apple_sleep_reg, 0); +} + +void apple_sleep_registry_ready(void) +{ + guard(mutex)(&apple_sleep_coprocs_lock); + WRITE_ONCE(apple_sleep_reg, apple_sleep_reg | APPLE_SLEEP_REG_READY); +} + +struct apple_sleep_coproc *apple_sleep_coproc_find(struct device *dev) +{ + int i; + + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev == dev) + return &apple_sleep_coprocs[i]; + + return NULL; +} + +/* Below EL2 a monitor owns the last mile, and this code doesn't run it. */ +int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, + int (*last_stop)(void *data), void *data) +{ + struct apple_sleep_coproc *c; + + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_coproc_find(dev)) + return -EEXIST; + c = apple_sleep_coproc_find(NULL); + if (!c) + return -ENOSPC; + c->asc = asc; + c->pcie_ans = pcie_ans; + c->last_check = NULL; + c->last_stop = last_stop; + c->data = data; + c->ready = false; + c->stopped = false; + c->first = false; + c->order = 0; + c->ps = 0; + c->dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register); + +int apple_sleep_coproc_register_first(struct device *dev, unsigned int order, + int (*last_stop)(void *data), void *data) +{ + struct apple_sleep_coproc *c; + struct device_node *np; + u32 ps = 0; + int ret; + + if (!last_stop || order >= APPLE_SLEEP_FIRST_ORDERS) + return -EINVAL; + np = of_parse_phandle(dev->of_node, "power-domains", 0); + if (np) { + of_property_read_u32(np, "reg", &ps); + of_node_put(np); + } + ret = apple_sleep_coproc_register(dev, NULL, false, last_stop, data); + if (ret) + return ret; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (c) { + c->first = true; + c->order = order; + c->ps = ps; + } + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_register_first); + +int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (!c) + return -ENOENT; + c->last_check = last_check; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_last_check); + +/* With no sleep controller bound, nothing would ever send the request: refuse it. */ +int apple_sleep_enclave_register(struct device *dev, int (*stop)(void *data), void *data) +{ + unsigned int reg; + + if (!is_kernel_in_hyp_mode()) + return -ENODEV; + + if (!stop) + return -EINVAL; + reg = READ_ONCE(apple_sleep_reg); + if (!(reg & APPLE_SLEEP_REG_BOUND)) + return -EOPNOTSUPP; + if (!(reg & APPLE_SLEEP_REG_ENCLAVE_ANY) && !apple_sleep_commit_stops_coprocs()) + return -EOPNOTSUPP; + + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_enclave.dev) + return -EBUSY; + apple_sleep_enclave.stop = stop; + apple_sleep_enclave.data = data; + apple_sleep_enclave.dev = dev; + + return 0; +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_register); + +void apple_sleep_enclave_unregister(struct device *dev) +{ + guard(mutex)(&apple_sleep_coprocs_lock); + if (apple_sleep_enclave.dev == dev) + memset(&apple_sleep_enclave, 0, sizeof(apple_sleep_enclave)); +} +EXPORT_SYMBOL_GPL(apple_sleep_enclave_unregister); + +void apple_sleep_coproc_unregister(struct device *dev) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (c) + memset(c, 0, sizeof(*c)); +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_unregister); + +void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + if (!c) + return; + c->ready = ready; + c->stopped = false; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_ready); + +bool apple_sleep_coproc_stopped(struct device *dev) +{ + struct apple_sleep_coproc *c; + + guard(mutex)(&apple_sleep_coprocs_lock); + c = apple_sleep_coproc_find(dev); + return c && c->stopped; +} +EXPORT_SYMBOL_GPL(apple_sleep_coproc_stopped); + +bool apple_sleep_commit_stops_coprocs(void) +{ + return (READ_ONCE(apple_sleep_reg) & APPLE_SLEEP_REG_READY) && is_kernel_in_hyp_mode() && + apple_sleep_power_loss(); +} +EXPORT_SYMBOL_GPL(apple_sleep_commit_stops_coprocs); + +/* + * The last thing that can fail on the way to the commit: no CPU but this one is up and + * interrupts are off, so no lock can be held. A coprocessor stopped here comes back only + * through a real sleep, so every refusal has to be behind this, and nothing after it may + * go back. In this order, on ANS itself: whatever its driver does last, + * status and control read, then 0 written twice. + */ +static void apple_sleep_coproc_run_off(struct apple_sleep_coproc *c) +{ + /* Its own last stop cleared its run bits. */ + if (!c->asc) { + c->stopped = true; + return; + } + c->status = readl(c->asc + ASC_CPU_STATUS); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); + c->stopped = true; +} + +/* A power word that a ready coprocessor's last stop lowers: on until the commit. */ +bool apple_sleep_coproc_lowers(u32 ps) +{ + int i; + + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) + if (apple_sleep_coprocs[i].dev && apple_sleep_coprocs[i].ready && + apple_sleep_coprocs[i].first && apple_sleep_coprocs[i].ps == ps) + return true; + return false; +} + +bool apple_sleep_enclave_registered(void) +{ + return apple_sleep_enclave.dev; +} + +void apple_sleep_registry_show(struct seq_file *s) +{ + int i; + + guard(mutex)(&apple_sleep_coprocs_lock); + seq_puts(s, "coprocessors registered for the commit's last stop:\n"); + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev) + continue; + seq_printf(s, " %s%s%s%s%s\n", dev_name(c->dev), c->pcie_ans ? " (ANS)" : "", + c->first ? " (first)" : "", c->last_check ? ", last check" : "", + c->ready ? ", ready" : ""); + } + seq_printf(s, "enclave sleep request left for the last refusal: %s\n", + apple_sleep_enclave.dev ? dev_name(apple_sleep_enclave.dev) : "none"); +} + +/* A registered coprocessor that isn't ready refuses the sleep. */ +unsigned int apple_sleep_coprocs_refuse(struct device *dev, bool *pcie_ans_idled) +{ + unsigned int refused = 0; + int i; + + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev) + continue; + if (c->ready) { + *pcie_ans_idled |= c->pcie_ans; + } else { + /* Nothing may lower a coprocessor's domain under its driver. */ + dev_err(dev, "refused: %s is running and its driver has not quiesced it\n", + dev_name(c->dev)); + refused++; + } + } + + return refused; +} + +/* The last refusals, each coprocessor's own: nothing has stopped yet. */ +int apple_sleep_coprocs_check(struct device *dev) +{ + int i, ret; + + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->stopped || !c->last_check) + continue; + ret = c->last_check(c->data); + if (ret) { + dev_err(dev, "refused: %s: %d\n", dev_name(c->dev), ret); + return ret; + } + } + + return 0; +} + +/* A nonzero result must mean no SLEEP was sent; post-send errors are fatal. */ +int apple_sleep_enclave_stop(struct device *dev) +{ + int ret; + + if (!apple_sleep_enclave.stop) + return 0; + ret = apple_sleep_enclave.stop(apple_sleep_enclave.data); + if (ret) + dev_err(dev, "refused before SEP SLEEP was sent: %d\n", ret); + + return ret; +} + +/* + * In two passes, after the last refusal: from the first stop only a real sleep restarts the + * GPU, so a failure here goes on to the commit. Those registered first stop whole first, + * before the off words drop under them: the GPU withdraws its PMP request there, and the + * PMP has to see that before its own stop. The rest after the match. With ANS among the ready + * ones, @pcie_ans. + */ +void apple_sleep_coprocs_stop(bool first, bool *pcie_ans) +{ + unsigned int order; + int i; + + if (first) + *pcie_ans = false; + /* Those registered first run by their order, the others in one pass over the table. */ + for (order = 0; order < (first ? APPLE_SLEEP_FIRST_ORDERS : 1); order++) { + for (i = 0; i < APPLE_SLEEP_COPROCS; i++) { + struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; + + if (!c->dev || !c->ready || c->first != first) + continue; + if (first && c->order != order) + continue; + *pcie_ans |= c->pcie_ans; + if (c->stopped) + continue; + c->stop_ret = c->last_stop ? c->last_stop(c->data) : 0; + apple_sleep_coproc_run_off(c); + } + } + dsb(sy); +} + +/* + * A controller that was tested with neither the lid's state at the wake nor the USB wake ports + * binds without the capability: the queries then answer as they do with no sleep driver. + */ +static bool apple_sleep_declines(unsigned int cap) +{ + unsigned int reg = READ_ONCE(apple_sleep_reg); + + return (reg & APPLE_SLEEP_REG_BOUND) && !(reg & cap); +} + +static unsigned long sleep_usb_wake_ports; + +void apple_sleep_usb_set_wake_ports(unsigned long ports) +{ + WRITE_ONCE(sleep_usb_wake_ports, ports); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_set_wake_ports); + +bool apple_sleep_usb_port_can_wake(unsigned int port) +{ + if (apple_sleep_declines(APPLE_SLEEP_REG_USB_WAKE_PORTS)) + return false; + + return port < BITS_PER_LONG && (READ_ONCE(sleep_usb_wake_ports) & BIT(port)); +} +EXPORT_SYMBOL_GPL(apple_sleep_usb_port_can_wake); + +/* Read in the SMC's noirq resume. Forgotten as each sleep begins. */ +static int sleep_lid_closed = -ENODATA; + +void apple_sleep_lid_at_wake(bool closed) +{ + WRITE_ONCE(sleep_lid_closed, closed); +} +EXPORT_SYMBOL_GPL(apple_sleep_lid_at_wake); + +int apple_sleep_lid_closed_at_wake(void) +{ + if (apple_sleep_declines(APPLE_SLEEP_REG_LID_AT_WAKE)) + return -ENODATA; + + return READ_ONCE(sleep_lid_closed); +} +EXPORT_SYMBOL_GPL(apple_sleep_lid_closed_at_wake); + +/* Set in the SMC's noirq resume. Forgotten as each sleep begins. */ +static int sleep_radio_woke = -ENODATA; + +void apple_sleep_radio_at_wake(bool woke) +{ + WRITE_ONCE(sleep_radio_woke, woke); +} +EXPORT_SYMBOL_GPL(apple_sleep_radio_at_wake); + +int apple_sleep_radio_woke(void) +{ + return READ_ONCE(sleep_radio_woke); +} +EXPORT_SYMBOL_GPL(apple_sleep_radio_woke); + +/* Each sleep begins with no answer about the wake before it. */ +void apple_sleep_wake_info_forget(void) +{ + WRITE_ONCE(sleep_lid_closed, -ENODATA); + WRITE_ONCE(sleep_radio_woke, -ENODATA); +} diff --git a/drivers/soc/apple/sleep-t6040.c b/drivers/soc/apple/sleep-t6040.c new file mode 100644 index 0000000000000..0efef072c7362 --- /dev/null +++ b/drivers/soc/apple/sleep-t6040.c @@ -0,0 +1,3168 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC system sleep: the T6040 (M4 Pro) part + * + * The T6040 sleeps through the same boot object and mailbox as the SoCs sleep.c takes from + * its lists, and nearly every step in between differs: three CPU complexes, five display + * pipes whose power states have to read off first, two storage PCIe lanes, the wake block + * and the order of the commit. So this file is a part of that driver (sleep-internal.h) with + * its own probe, state and suspend operations. The commit follows the T6040 entry as + * measured on the J616s, with what the SoC's sleep engine needs from the OS (see + * t6040_sleep_commit); the wake return mirrors the PMGR wake restore + * (see t6040_sleep_return). The registry of coprocessors and of the enclave + * (linux/soc/apple/sleep.h), the dry run level and the mailbox layout are the generic + * driver's. + * + * It refuses first: + * + * - It binds below EL2 and without the boot object's mailbox as well, and shows both as + * blockers in debugfs t6040-sleep/preflight, next to the CPUs that can park and the + * display power states the ADT lists to check before a sleep (device-check-ps-b4-sleep). + * - It offers suspend-to-RAM only to a system booted with t6040_sleep.allow_entry=1. + * Without it suspend-to-idle stays the default sleep, and it passes through here + * untouched either way. + * - Its suspend_ops .begin computes the blocker list of a suspend-to-RAM. With a blocker + * left it prints every one and returns an error before any device suspends. + * - Dry runs (t6040_sleep.dry_run, also writable at runtime): + * 1: pm_suspend_via_firmware() is set, so every driver's firmware-driven + * sleep step that can be taken back runs (apple_sleep_power_loss() is + * false, so what only a power cycle restores is left out), the CPUs + * park and answer the commit's hold, the commit point's checks run, + * and the sleep is refused there and resumed warm. + * 2: 1, plus the last steps that can still be taken back (each + * coprocessor's last check, such as ANS's check of the shutdown + * notification it sent at its suspend). The SMC's sleep commit is not + * one of them: the SMC leaves the state it puts it in only through a + * power loss of the SoC, so no dry run sends it. + * The AON watchdog's WD2 is armed from .begin to .end of a dry run + * (t6040_sleep.wd2_secs, 0 to leave it off). + * - The real commit (t6040_sleep_commit) runs only when no blocker remains, + * t6040_sleep.allow_entry=1 is given and no dry run is asked for. The AON + * watchdog's WD2 is its net (t6040_sleep.sleep_wd_secs), and over the device + * suspend and resume around it. + * + * The lid's state at the wake and the USB wake ports, which the generic driver keeps for the + * display and the port controllers, are declined: this sleep was tested without them. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "sleep-internal.h" + +/* The parameters keep the names they had when this was a driver of its own. */ +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "t6040_sleep." + +#define PS_STRIDE 8 +#define PS_MAX_PER_GROUP 16 +#define PS_SIO_FIRST 0x150 /* SOC-East: I2C0, the first SIO peripheral word */ +/* The extent of the two power word pages the driver accesses (the DT window sizes). */ +#define SOC_EAST_SIZE 0xd000 +#define SOC_PS_SIZE 0x9000 +#define PS_SIO_LAST 0x228 /* SOC-East: UART6, the last */ + +/* + * ANS's storage PCIe block: the two ST lanes (apcie_st0/apcie_st1) and their + * shared PHY (T8140 has a single lane). Taken down at the commit + * and raised on wake. The power words sit in SOC-East (t604x-pmgr.dtsi + * offsets, same bank as ps_ans); the control windows (axi2af/cio3pll/clkgen/ + * phy-block/phy-ip/rc) and the PLL/lane tunables are not in this SoC's static + * description and are published by the boot object (aura) under the apple,apcie + * node, as on the T8140. The block-internal offsets are the T8140's + * (sleep-internal.h): the same Apple PCIe IP. + */ +#define PS_ANS 0x370 /* SOC-East: ps_ans */ +#define PS_APCIE_ST0 0x108 /* SOC-East: ps_apcie_st0 */ +#define PS_APCIE_ST1 0x110 /* SOC-East: ps_apcie_st1 */ +#define PS_APCIE_SYS_ST0 0x380 /* SOC-East: ps_apcie_sys_st0 */ +#define PS_APCIE_SYS_ST1 0x388 /* SOC-East: ps_apcie_sys_st1 */ +#define PS_APCIE_PHY_SW 0x398 /* SOC-East: ps_apcie_phy_sw (the shared PHY) */ +#define PS_FAB2_SOC 0x338 /* SOC: the ATC commons' parent */ +/* The clock-gated words (TARGET 4) the commit writes, as the sleep engine asks for them. */ +#define PS_APCIE_PHY_SW_CLKGATED 0x0f000244 +#define PS_APCIE_GP_CLKGATED 0x00000244 + +/* + * cluster-ctl (ADT pmgr reg[0] + cluster-ctl-offset 0x20000 = 0x502820000), one word per + * complex, measured on the J616s: the CPU complexes E +0x0, P0 +0x4 and P1 +0x8, then the + * internal and the external display's complexes +0xc and +0x10. The low nibble 0 powers a + * complex down and 0xf powers it up; BIT4 reads set until a power-down has finished. + */ +#define CLUSTER_CTL(n) ((n) * 4) +#define CLUSTER_CTL_NR 5 /* E, P0, P1, internal and external display */ +#define CLUSTER_CTL_FIRST_WAIT 3 /* the two display complexes are polled */ +#define CLUSTER_CTL_BUSY BIT(4) +#define CLUSTER_CTL_NIBBLE GENMASK(3, 0) +#define CLUSTER_CTL_ON 0xf +#define CLUSTER_TIMEOUT_US 50000 + +/* + * SOC-PwrGate (ADT pmgr reg[0x26] = 0x50282c000): the retention bits at +0x0/+0x40 and + * the three CPM voters at +0x100/+0x108/+0x110, each BIT31. The wake restore re-asserts + * them. The entry leaves them alone: the voters already read 0x80000000 there. + */ +#define PWRGATE_ON 0x80000000 +#define PWRGATE_WAKE_0 0x000 +#define PWRGATE_WAKE_1 0x040 +#define PWRGATE_CPM_0 0x100 +#define PWRGATE_CPM_1 0x108 +#define PWRGATE_CPM_2 0x110 + +/* + * SOC-East (ADT pmgr reg[2] = 0x502280000). The SEP/retention word at +0xc00: the wake + * clears PS_DEV_DISABLE and then polls PS_BUSY clear before going on. Also holds several + * of the axi2af domains' power words. + */ +#define SOC_EAST_RETENTION 0xc00 +#define SOC_EAST_TIMEOUT_US 5000 + +/* + * The clock quiesce, whole and nothing else: two SOC-East words lose BIT28 and are + * polled until [7:4] reads their [3:0]; then the four clk-mux park words of ClkGen SOC (ADT + * pmgr reg[3] = 0x500000000, +0x38050/054/060/064), in t6040_clkgen_mux[] order, get [27:24] + * := 0 with BIT31 set, each polled until BIT30 clears. The wake writes the saved clk-mux + * words back; there is nothing else to restore. Not measured on the J616s. + */ +#define SOC_EAST_CLK_A 0x120 +#define SOC_EAST_CLK_B 0x130 +#define SOC_EAST_CLK_AUTO BIT(28) +#define CLKGEN_MUX_SET 0x80000000 +#define CLKGEN_MUX_MASK 0xf0ffffff +#define CLKGEN_MUX_WORDS 4 +#define CLKGEN_MUX_PENDING BIT(30) +#define CLK_QUIESCE_TIMEOUT_US 1000 + +/* + * The keep-alive enables: 32 words in PMGR SOC from +0x1dc000 and 32 in SOC-East from + * +0x50000, stride 0x44, BIT0 the keep-alive. Cleared right before the S2R timer, set + * again at the wake. + */ +#define T6040_KA_SOC_PA 0x5029dc000ULL +#define T6040_KA_EAST_PA 0x5022d0000ULL +#define KA_WORDS 32 +#define KA_STRIDE 0x44 +#define KA_SIZE (KA_WORDS * KA_STRIDE) +#define KA_ENABLE BIT(0) + +/* + * The SMC's AP watchdog deadline (the watchdog timer's reg[2] + 8): written 0 right after + * the AON WD2 is armed for the sleep. + */ +#define T6040_SMC_WDT_DEADLINE_PA 0x508330008ULL + +/* + * The AGX residency gate at the top of the PMGR quiesce: unless + * MiniPMGR +0x75010 has both [19:18] set (it read 0 on the J616s), the GPU's AFR power + * word (pmgr_gfx 0x288e80000 + 0x100, ADT power-device AFR) is lowered when its + * target is not 0: [19:16] := 4 and polled, [7:4] polled to 4, then the word := 0 + * and [7:4] polled to 0. + */ +#define T6040_AGX_GATE_PA 0x508375010ULL +#define AGX_GATE_RESIDENT GENMASK(19, 18) +#define T6040_AFR_PS_PA 0x288e80100ULL +#define AGX_TIMEOUT_US 10000 + +/* + * Before the SoC-specific steps of the PMGR quiesce, in order: + * - the ACC quiesce of c = E, P0, P1: APSC off, ACC +0xe20020 BIT23 := 1 (64-bit), + * poll BIT7 clear (CPM gating after it reads back already set: not written); + * - features off: cpu-fixed-freq-pll-relock, BIT42 := 0 in the E and P0 + * ACC words; then the TVMs disp2, afr, dcs (two), gfx (two), disp, soc: control + * BIT0 := 0, status polled until BIT1 clears. + * The PMC's DCS floor vote is not touched: an access to that word hangs the bus. Every poll + * here is bounded. + */ +static const u64 t6040_acc_apsc_pa[] = { 0x210e20020ULL, 0x211e20020ULL, 0x212e20020ULL }; +#define ACC_APSC_DISABLE BIT_ULL(23) +#define ACC_APSC_BUSY BIT_ULL(7) +#define ACC_PLL_RELOCK BIT_ULL(42) +#define ACC_PLL_RELOCK_COMPLEXES 2 +#define QUIESCE_TIMEOUT_US 2000 + +static const struct { + u64 ctl; + u64 status; +} t6040_tvm[] = { + { 0x502920000ULL, 0x502920250ULL }, /* disp2-tvm */ + { 0x288e95000ULL, 0x288e953e0ULL }, /* afr-tvm */ + { 0x502908000ULL, 0x502908250ULL }, /* dcs-tvm */ + { 0x502910000ULL, 0x502910250ULL }, + { 0x288e94000ULL, 0x288e943e0ULL }, /* gfx-tvm */ + { 0x288e94400ULL, 0x288e947e0ULL }, + { 0x502918000ULL, 0x502918250ULL }, /* disp-tvm */ + { 0x502900000ULL, 0x502900250ULL }, /* soc-tvm */ +}; + +#define TVM_ENABLE BIT(0) +#define TVM_BUSY BIT(1) + +/* + * The PMC's bandwidth-request and boost interface enables of the display pipes + * (0x5048ac020 and +0x8), one bit per pipe. + */ +#define T6040_PMC_BWR_PA 0x5048ac020ULL +#define PMC_BOOST 0x8 + +/* + * The PMU over SPMI (spmi = 0x509014000, spmi-a0 = 0x509188000), written as on the T8140. + * Measured on the J616s: the boot stage is set to 0x50 by an extended write to + * PMU_BOOT_STAGE, and NOTIFY_ENABLE is 0x2000001 on both controllers' +0x40. + */ +/* The PMU's USID on the sleep's SPMI bus, and the boot stage that announces a sleep */ +#define PMU_USID 0xe +#define PMU_BOOT_STAGE 0xf801 /* T6040 slave/reg */ +#define PMU_BOOT_STAGE_SLEEP 0x50 +/* The response vector's enable for the sleep */ +#define SPMI_NOTIFY_ENABLE_SLEEP 0x2000001 + +/* + * MasterMini, the block of the always-on PMGR (ADT pmgr reg[0x37] = 0x508900000). + * The S2R timer at +0x4000 (physical 0x508904000) is the commit: writing 1 arms + * the SoC sleep engine. Measured on the J616s: exactly this write. + */ +#define MASTERMINI_S2R_TIMER 0x4000 + +/* + * AOP-global (the AP controls, window 0x508000000). Mapped from the + * S2R-control word 0x508004000: +0x0 the S2R control (cleared to 0 on wake), + * +0xc..+0x2c the 9 wake-source masks (armed before + * sleep), +0x30..+0x50 the 9 wake-source status words (read on wake). + */ +#define AOP_S2R_CTRL 0x00 +#define AOP_WAKE_MASK(w) (0x0c + (w) * 4) +#define AOP_WAKE_STATUS(w) (0x30 + (w) * 4) +#define AOP_WAKE_WORDS 9 + +/* axi2af fabric-bridge retention re-arm. */ +#define AXI2AF_BRIDGE_104 0x104 +#define AXI2AF_BRIDGE_108 0x108 + +/* The internal display: its PMGR words and its DART. */ +#define T6040_DCP_DART_PA 0x402340000ULL +#define T6040_DISP_PWRGATE_PA 0x5022a1000ULL +#define DISP_PWRGATE_A 0x1c /* := 3 before DISP_SYS goes down, 0 after it is up */ +#define DISP_PWRGATE_B 0x60 /* := 0 before DISP_SYS goes down, 1 after it is up */ +#define PS_DISP_FE 0x368 /* soc_east */ +#define PS_DISP_SYS 0x258 /* soc_east */ +#define DISP_PS_TIMEOUT_US 8000 +#define DISP_CPU_GATE_US 50000 + +/* The GP PCIe (t602x port layout, as T8140's radio port): power words in soc_east. */ +#define PS_APCIE_GP 0x100 +#define PS_APCIE_SYS_GP 0x390 +#define GP_PIN_DATA BIT(0) +/* The PMC's DCP voters: the count of DCPs powered (+0) and a second voter (+4), both 0 at sleep */ +#define T6040_PMC_DCP_PA 0x5048a9c14ULL + +/* The axi2af fabric-bridge retention domains, t6040_axi2af[] */ +enum { + AXI2AF_ATC0_COMMON, + AXI2AF_ATC1_COMMON, + AXI2AF_ATC2_COMMON, + AXI2AF_ATC3_COMMON, + AXI2AF_DISP_SYS, + AXI2AF_DISPEXT0_SYS, + AXI2AF_DISPEXT1_SYS, + AXI2AF_DISPEXT2_SYS, + AXI2AF_DISPEXT3_SYS, + AXI2AF_ANS, + AXI2AF_AVD_SYS, + AXI2AF_VENC0_SYS, + AXI2AF_VENC1_SYS, + AXI2AF_JPG, + AXI2AF_APCIE_SYS_GP, + T6040_AXI2AF_DOMAINS +}; + +/* + * The AON watchdog's WD2, the sleep safety net: ADT wdt reg[0] = 0x50836c000, with the + * T8140's registers. Linux's own watchdog driver runs WD1 (+0x10..+0x1c), not this one. + * ALARM is 32 bits at 24 MHz. + */ +#define WDT_MAX_SECS 178U +#define T6040_WDT_PA 0x50836c000ULL /* the address t6040_resume_stub writes */ + +struct t6040_sleep_group { + const char *name; + struct regmap *regmap; + u32 first; /* the pipe's CPU */ + u32 last; /* its last sub-block */ +}; + +/* A device the commit point looks at: bound or not, and its own power word. */ +struct t6040_sleep_watch { + const char *compatible; + struct regmap *ps_regmap; + u32 ps_offset; + bool has_ps; + bool bound; +}; + +enum { + T6040_WATCH_SEP, + T6040_WATCH_ANS, + T6040_WATCH_COUNT, +}; + +/* + * ANS's storage-apcie control windows and PLL/lane tunables, mapped from the + * apple,apcie node the boot object publishes (t6040_sleep_map_apcie). Every + * pointer is NULL until a loader fills the node: @valid is false then, and a + * real sleep with ANS stopped refuses. The two ST lanes share this one PHY. + */ +struct t6040_apcie_st { + void __iomem *axi; /* reg-name "axi2af" (shared) */ + void __iomem *cio3pll; /* reg-name "cio3pll" (shared) */ + void __iomem *clkgen; /* reg-name "clkgen" (shared) */ + void __iomem *phy; /* reg-name "phy-block" (shared PHY) */ + void __iomem *phy_ip; /* reg-name "phy-ip" */ + void __iomem *rc; /* reg-name "rc" */ + struct apple_tunable *axi_tunable; /* apple,tunable-axi2af */ + struct apple_tunable *cio3pll_tunable; /* apple,tunable-cio3pll */ + struct apple_tunable *clkgen_tunable; /* apple,tunable-clkgen */ + struct apple_tunable *phy_tunable; /* apple,tunable-phy */ + struct apple_tunable *phy_ip_pll; /* apple,tunable-phy-ip-pll */ + struct apple_tunable *phy_ip_lanes; /* apple,tunable-phy-ip-auspma */ + bool valid; +}; + +struct t6040_sleep { + struct device *dev; + struct t6040_sleep_group *groups; + unsigned int ngroups; + /* The dry run level of the suspend in progress, from .begin to .end; 0 otherwise. */ + unsigned int dry; + void __iomem *cpu_start; + void __iomem *wdt; + /* The DT's wdt block is the one t6040_resume_stub hardcodes. */ + bool wdt_stub_ok; + /* The commit's and the wake's SoC blocks (see the #defines). Optional. */ + void __iomem *cluster_ctl; + void __iomem *pwrgate; + void __iomem *dcp_dart; /* dart-dcp, for its transaction idle wait */ + void __iomem *disp_pwrgate; /* internal display retention window 0x5022a1000 */ + bool disp_down; /* t6040_disp_off() lowered the internal display */ + void __iomem *pmc_dcp; /* PMS pmc-dcp voter pair 0x5048a9c14/c18 */ + u32 pmc_dcp_saved[2]; + u32 pmc_if_saved[2]; /* BWR and boost interface enables at the commit */ + bool pmc_if_valid; + /* The GP PCIe host's two ports (Wi-Fi/BT, SD) for the commit's port stop (T8140's). */ + void __iomem *gp_port[2]; + void __iomem *gp_phy[2]; + void __iomem *gp_pins; + int gp_perst[2]; + bool gp_perst_low[2]; + void __iomem *clkgen; + void __iomem *spmi0; + void __iomem *spmi_a0; + void __iomem *mastermini; + void __iomem *dcs[16]; /* DCS(n,k): 0x220200000 + n * 0x2000000 + k * 0x200000 */ + void __iomem *atc_cfg[4]; /* ATCPHY_CFG(n) 0x383200000 + n * 0x8000000 */ + void __iomem *aop_ps; /* MiniPMGR AOP + audio island PS words 0x508304000..0x158 */ + u32 sio_ps_saved[(PS_SIO_LAST - PS_SIO_FIRST) / PS_STRIDE + 1]; + bool sio_ps_saved_valid; + /* The SOC-East and SOC power word pages: fabrics, ATC commons, DCS, SIO. */ + void __iomem *soc_east; /* 0x502280000 */ + void __iomem *soc_ps; /* 0x502800000 */ + void __iomem *aop_global; + void __iomem *axi2af_bridge[T6040_AXI2AF_DOMAINS]; + struct aura_s2r_mailbox *mbox; + u64 mbox_pa; + bool power_ops; + bool wd2_armed; + /* Set by .begin for a suspend to RAM; .end only undoes what .begin did. */ + bool mem_begun; + u64 siq_cfg; + u64 cntkctl; + /* Saved across the commit, restored on wake. */ + u32 clkgen_mux_saved[CLKGEN_MUX_WORDS]; + bool clkgen_saved; + void __iomem *ka_soc; + void __iomem *ka_east; + u64 ka_on_before; /* keep-alive bits set before the commit cleared them */ + void __iomem *smc_wdt_deadline; + void __iomem *agx_gate; + void __iomem *s2r_lo_count; + void __iomem *acc_apsc[3]; + void __iomem *tvm_ctl[8]; + void __iomem *tvm_status[8]; + void __iomem *pmc_bwr; + void __iomem *afr_ps; + u32 cluster_saved[CLUSTER_CTL_NR]; + u32 wake_mask_saved[AOP_WAKE_WORDS]; + bool wake_mask_saved_valid; + u32 wake_status[AOP_WAKE_WORDS]; + bool returned; + /* ANS's storage ST lanes: windows + whether a commit stopped ANS. */ + struct t6040_apcie_st apcie_st; + bool pcie_ans_stopped; + int pcie_ans_on_ret; + u32 apcie_wake_step; + u32 gp_phy_saved[2], gp_appclk_saved[2], gp_perst_saved[2]; + bool gp_saved[2]; + unsigned int gp_clk_timeouts; + unsigned int aop_ps_timeouts; + u32 aop_ps_saved[44]; + bool aop_ps_saved_valid; + struct t6040_sleep_watch watch[T6040_WATCH_COUNT]; +}; + +/* + * 0: no dry run. 1: the firmware-driven device suspend short of what only a power + * cycle restores, parking and the commit's hold, refused at the commit point. 2: 1 and + * the last steps that can still be taken back. Read at each suspend's .begin. + */ +static unsigned int dry_run; +module_param(dry_run, uint, 0644); +MODULE_PARM_DESC(dry_run, + "Dry run level: 1 firmware-driven device sleep refused at the commit point, 2 also the last steps"); + +static bool allow_entry; +module_param(allow_entry, bool, 0444); +MODULE_PARM_DESC(allow_entry, + "Reach the real commit if no blocker remains"); + +/* + * The AON WD2 net armed at the commit. 5 s covers the reconfig; a longer + * net tells "the SoC never slept" (reset at the net) from "it slept and the wake + * failed" (reset at the wake source). Never off: 1 to 178. + */ +static unsigned int sleep_wd_secs = 30; +module_param(sleep_wd_secs, uint, 0644); +MODULE_PARM_DESC(sleep_wd_secs, + "AON WD2 net armed at the S2R commit, in seconds (default 30, clamped to 1..178)"); + +static unsigned int wd2_secs = WDT_SUSPEND_SECS; +module_param(wd2_secs, uint, 0644); +MODULE_PARM_DESC(wd2_secs, + "Arm the AON WD2 for this many seconds from .begin to .end of a dry run (0: off, at most 178)"); + +static struct t6040_sleep *t6040_sleep; + +static void t6040_iounmap(void *p) +{ + iounmap((void __iomem __force *)p); +} + +/* + * Every window here non-posted (nGnRnE), as the PMGR, PMC and coprocessor blocks + * require. No region request: these overlap other drivers. + */ +static void __iomem *t6040_ioremap_np(struct device *dev, phys_addr_t pa, size_t size) +{ + void __iomem *p = ioremap_np(pa, size); + + if (!p || devm_add_action_or_reset(dev, t6040_iounmap, (void __force *)p)) + return NULL; + return p; +} + +/* The generic driver answers apple_sleep_power_loss() by it. */ +static void t6040_sleep_set_dry(struct t6040_sleep *s, unsigned int dry) +{ + s->dry = dry; + apple_sleep_set_dry(dry); +} + +/* + * In the power-state check, a sub-block is on while its TARGET is not 0. A + * pipe's CPU word never counts: it stays under automatic power gating (taking it out + * of automatic mode raises an SError) and the SoC power-down drops it. + */ +static bool t6040_sleep_ps_on(u32 val, bool cpu) +{ + return !cpu && FIELD_GET(PS_TARGET, val); +} + +static unsigned int t6040_sleep_show_group(struct seq_file *m, const struct t6040_sleep_group *g) +{ + unsigned int on = 0; + u32 off; + + for (off = g->first; off <= g->last; off += PS_STRIDE) { + bool cpu = off == g->first; + u32 val; + int ret; + + ret = regmap_read(g->regmap, off, &val); + if (ret) { + /* Unreadable counts as on: the refusal fails closed. */ + if (m) + seq_printf(m, " %-12s +%#05x read failed (%d)\n", g->name, off, + ret); + on++; + continue; + } + if (t6040_sleep_ps_on(val, cpu)) + on++; + if (m) + seq_printf(m, " %-12s +%#05x %s %#010x target %lu actual %lu%s\n", g->name, + off, cpu ? "cpu " : "child", val, FIELD_GET(PS_TARGET, val), + FIELD_GET(PS_ACTUAL, val), + t6040_sleep_ps_on(val, cpu) ? " ON" : ""); + } + + return on; +} + +/* How many secondaries can leave the kernel (park), and the boot ops' name. */ +static unsigned int t6040_sleep_parkable(const char **name) +{ + unsigned int cpu, parkable = 0; + + for_each_possible_cpu(cpu) { + const struct cpu_operations *ops = get_cpu_ops(cpu); + + if (cpu && ops && ops->cpu_die && ops->cpu_disable) + parkable++; + } + if (name) + *name = get_cpu_ops(0) ? get_cpu_ops(0)->name : "?"; + + return parkable; +} + +/* + * The live blockers, recomputed each time: not at EL2, no boot-object return + * path, a CPU that cannot park, or a display pipe still on. Returns the count, + * and lists each when @s is given. @disp_out, if not NULL, gets the subtotal of + * display-pipe blockers -- these clear during device suspend (apple-dcp's + * .suspend stops the DCP and lowers its domains), so .begin excludes them and + * only .enter, after the device suspend, enforces them. + */ +static unsigned int t6040_sleep_live_blockers(struct t6040_sleep *s, struct seq_file *m, + unsigned int *disp_out) +{ + /* Mapped and aura's: after a resume from disk, this boot's (t6040_sleep_restore()). */ + bool mailbox = s->mbox; + bool el2 = is_kernel_in_hyp_mode(); + unsigned int i, blockers = 0, disp = 0, parkable; + const char *ops_name; + + parkable = t6040_sleep_parkable(&ops_name); + + if (m) { + seq_printf(m, "kernel at EL2: %s\n", el2 ? "yes" : "no"); + seq_printf(m, "boot object return path (/chosen/aurora,s2r-mailbox): %s\n", + mailbox ? "yes" : "no"); + seq_printf(m, "secondaries that can leave the kernel: %u of %u (%s)\n", parkable, + num_possible_cpus() - 1, ops_name); + } + blockers += !el2 + !mailbox + (parkable != num_possible_cpus() - 1); + + if (m) + seq_puts(m, "power states checked before sleep (device-check-ps-b4-sleep):\n"); + for (i = 0; i < s->ngroups; i++) { + unsigned int on = t6040_sleep_show_group(m, &s->groups[i]); + + if (m) + seq_printf(m, " %-12s %u on\n", s->groups[i].name, on); + disp += !!on; + } + + if (disp_out) + *disp_out = disp; + return blockers + disp; +} + +/* The display groups' on-state, one line per group, for the log at the commit point. */ +static unsigned int t6040_sleep_log_groups(struct t6040_sleep *s) +{ + unsigned int i, blockers = 0; + + for (i = 0; i < s->ngroups; i++) { + const struct t6040_sleep_group *g = &s->groups[i]; + u32 off, on_mask = 0, val; + unsigned int bit = 0; + + for (off = g->first; off <= g->last; off += PS_STRIDE, bit++) { + if (regmap_read(g->regmap, off, &val) || + t6040_sleep_ps_on(val, off == g->first)) { + on_mask |= BIT(bit); + dev_info(s->dev, " %s +%#x = %#x\n", g->name, off, val); + } + } + if (on_mask) + dev_info(s->dev, "%-8s ON (on mask %#x, cpu first)\n", g->name, + on_mask); + blockers += !!on_mask; + } + + return blockers; +} + +/* Log the blockers outside the display, for a refused or dry .begin. */ +static void t6040_sleep_log_blockers(struct t6040_sleep *s) +{ + const char *ops_name; + + dev_info(s->dev, "EL2 %s, s2r-mailbox %s, parkable %u/%u (%s)\n", + is_kernel_in_hyp_mode() ? "yes" : "no", s->mbox ? "yes" : "no", + t6040_sleep_parkable(&ops_name), num_possible_cpus() - 1, ops_name); +} + +/* + * CPU parking power ops. MPIDR Aff1 is the cluster (E=0, P0=1, + * P1=2), Aff0 the core. The start and stop bit is the core's bit in the ADT + * function-enable_core masks, which is its cpu-id: the E cluster has four + * slots (bits 0-3), each P cluster six (P0 bits 4-9, P1 bits 10-15; one slot + * of each is harvested on J616s). Four-core banks (4 * cluster + core) would + * give P1 core 0 the bit of P0 core 4. + */ +static const u8 t6040_cluster_first_bit[] = { 0, 4, 10 }; +static const u8 t6040_cluster_slots[] = { 4, 6, 6 }; + +static unsigned int t6040_cpu_cluster(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 1); +} + +static unsigned int t6040_cpu_core(u64 mpidr) +{ + return MPIDR_AFFINITY_LEVEL(mpidr, 0); +} + +static u32 t6040_cpu_start_bit(u64 mpidr) +{ + unsigned int cluster = t6040_cpu_cluster(mpidr), core = t6040_cpu_core(mpidr); + + if (WARN_ON_ONCE(cluster >= ARRAY_SIZE(t6040_cluster_first_bit) || + core >= t6040_cluster_slots[cluster])) + return 0; + return BIT(t6040_cluster_first_bit[cluster] + core); +} + +/* + * Each core's CoreSight external debug block (ADT cpuN coresight-reg: 0x210010000 + + * cluster * 0x1000000 + core * 0x100000). Every CPU, boot and secondary, + * unlocks it (EDLAR) and writes EDPRCR := 0 right before its deep WFI, so that + * the debug logic does not prevent power gating. With + * EDPRCR.CORENPDRQ left set, a core only emulates its power-down. + */ +#define T6040_ED_BASE_PA 0x210010000ULL +#define T6040_ED_CLUSTER_STRIDE 0x1000000ULL +#define T6040_ED_CORE_STRIDE 0x100000ULL +#define ED_EDPRCR 0x310 +#define ED_EDLAR 0xfb0 +#define ED_LOCK_ACCESS_KEY 0xc5acce55 +#define T6040_ED_CLUSTERS ARRAY_SIZE(t6040_cluster_slots) +#define T6040_ED_CORES 6 /* the most slots of a cluster */ + +static void __iomem *t6040_ed[T6040_ED_CLUSTERS][T6040_ED_CORES]; + +static void t6040_ed_map(struct device *dev) +{ + unsigned int cpu, n = 0; + + for_each_possible_cpu(cpu) { + u64 mpidr = cpu_logical_map(cpu); + unsigned int c = t6040_cpu_cluster(mpidr), k = t6040_cpu_core(mpidr); + + if (c >= T6040_ED_CLUSTERS || k >= T6040_ED_CORES || t6040_ed[c][k]) + continue; + t6040_ed[c][k] = t6040_ioremap_np(dev, T6040_ED_BASE_PA + + c * T6040_ED_CLUSTER_STRIDE + + k * T6040_ED_CORE_STRIDE, 0x1000); + n += !!t6040_ed[c][k]; + } + dev_dbg(dev, "external debug windows mapped for %u cores\n", n); +} + +static void t6040_ed_release(void) +{ + u64 mpidr = read_cpuid_mpidr(); + unsigned int c = t6040_cpu_cluster(mpidr), k = t6040_cpu_core(mpidr); + void __iomem *ed; + + if (c >= T6040_ED_CLUSTERS || k >= T6040_ED_CORES) + return; + ed = t6040_ed[c][k]; + if (!ed) + return; + writel_relaxed(ED_LOCK_ACCESS_KEY, ed + ED_EDLAR); + writel_relaxed(0, ed + ED_EDPRCR); +} + +static void t6040_deep_wfi_arm(void) +{ + /* A timer left running ends the WFI and costs power until reset. */ + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntp_ctl_el0); + write_sysreg(ARCH_TIMER_CTRL_IT_MASK, cntv_ctl_el0); + t6040_ed_release(); + sysreg_clear_set_s(SYS_APL_DEEP_SLEEP, 0, DEEP_SLEEP_ENABLE); +} + +/* A core's last instructions: nothing brings it back but a reset. */ +static void __noreturn t6040_deep_wfi(void) +{ + t6040_deep_wfi_arm(); + + while (true) { + dsb(sy); + isb(); + wfi(); + write_sysreg_s(IPI_SR_PENDING, SYS_APL_IPI_SR_EL1); + } +} + +static void t6040_cpu_stop(u64 mpidr) +{ + u32 bit = t6040_cpu_start_bit(mpidr); + + if (t6040_sleep && t6040_sleep->cpu_start && bit) + writel(bit, t6040_sleep->cpu_start + CPU_START_STOP); + dsb(sy); +} + +static void t6040_cpu_off(u64 mpidr) +{ + t6040_deep_wfi(); +} + +static int t6040_cpu_on(u64 mpidr) +{ + struct t6040_sleep *s = t6040_sleep; + unsigned int cluster = t6040_cpu_cluster(mpidr); + u32 bit = t6040_cpu_start_bit(mpidr); + + if (!s) + return -ENODEV; + if (!bit) + return -EINVAL; + writel(bit, s->cpu_start + CPU_START_SYS); + writel(BIT(t6040_cpu_core(mpidr)), s->cpu_start + CPU_START_CLUSTER(cluster)); + + return 0; +} + +static const struct spin_table_power_ops t6040_cpu_power_ops = { + .cpu_stop = t6040_cpu_stop, + .cpu_off = t6040_cpu_off, + .cpu_on = t6040_cpu_on, +}; + +/* + * The commit's hold needs them. Registered, they also let kexec take parking + * CPUs as able to leave the kernel, for the rest of this boot. Nothing calls + * @cpu_stop or @cpu_off but a commit, and @cpu_on starts only a CPU a commit + * stopped. + */ +static void t6040_sleep_power_ops(struct t6040_sleep *s) +{ + if (s->power_ops) + return; + if (!spin_table_set_power_ops(&t6040_cpu_power_ops)) { + s->power_ops = true; + dev_dbg(s->dev, "CPU parking power ops registered\n"); + } +} + +static void t6040_sleep_set_wd2(struct t6040_sleep *s, unsigned int secs) +{ + apple_sleep_wd2_arm(s->wdt, secs); + s->wd2_armed = true; +} + +static void t6040_sleep_clear_wd2(struct t6040_sleep *s) +{ + if (!s->wd2_armed) + return; + writel(0, s->wdt + WDT_WD2_CTL); + s->wd2_armed = false; +} + +/* + * In process context, after the device suspend: which watched devices are + * bound. The commit point runs with interrupts off and only reads power words. + */ +static void t6040_sleep_watch_bound(struct t6040_sleep *s) +{ + int i; + + for (i = 0; i < T6040_WATCH_COUNT; i++) { + struct t6040_sleep_watch *w = &s->watch[i]; + struct device_node *np = of_find_compatible_node(NULL, NULL, w->compatible); + struct platform_device *pdev = np ? of_find_device_by_node(np) : NULL; + + w->bound = pdev && device_is_bound(&pdev->dev); + if (pdev) + put_device(&pdev->dev); + of_node_put(np); + } +} + +/* Its power word's actual state is on. Unknown counts as on. */ +static bool t6040_sleep_watch_on(struct t6040_sleep_watch *w, u32 *val) +{ + *val = 0; + if (!w->has_ps || regmap_read(w->ps_regmap, w->ps_offset, val)) + return true; + return FIELD_GET(PS_ACTUAL, *val); +} + +/* + * The checks a real commit makes, read-only: every registered coprocessor + * quiesced by its driver, one CPU up and it the boot object's, the SEP bound + * with its sleep request left here, ANS stopped only by its driver, and the + * display pipes off. Logs each failure and returns the count. + */ +static unsigned int t6040_sleep_refuse(struct t6040_sleep *s) +{ + struct t6040_sleep_watch *sep = &s->watch[T6040_WATCH_SEP]; + struct t6040_sleep_watch *ans = &s->watch[T6040_WATCH_ANS]; + bool ans_idled = false; + unsigned int refused; + u32 val; + + refused = apple_sleep_coprocs_refuse(s->dev, &ans_idled); + + if (num_online_cpus() != 1 || !s->mbox || + (read_cpuid_mpidr() & MPIDR_HWID_BITMASK) != s->mbox->boot_mpidr) { + dev_err(s->dev, "refused: wrong CPU, other CPUs online, or no mailbox\n"); + refused++; + } + + if (!sep->bound) { + /* Nobody told it of the sleep: a commit with it running never returns. */ + t6040_sleep_watch_on(sep, &val); + dev_err(s->dev, "refused: sep isn't bound (state %#x)\n", val); + refused++; + } else if (!apple_sleep_enclave_registered()) { + dev_err(s->dev, "refused: sep's sleep request isn't left for the last refusal\n"); + refused++; + } + + /* Only ANS's driver can stop it, and it doesn't come back without a real sleep. */ + if (!ans_idled && t6040_sleep_watch_on(ans, &val)) { + dev_err(s->dev, "refused: ANS is on (%#x) with no driver to stop it\n", val); + refused++; + } + + /* + * ANS would be stopped, but its ST lanes have no way back without the + * apple,apcie control windows + PLL/lane tunables: refused. + */ + if (ans_idled && !s->apcie_st.valid) { + dev_err(s->dev, "refused: ANS would stop but the storage-apcie windows/tunables are absent (apple,apcie)\n"); + refused++; + } + + /* The engine waits for the display CPUs with no timeout. */ + if (t6040_sleep_log_groups(s)) + refused++; + + return refused; +} + +/* + * The AOP and audio island's power words (MiniPMGR 0x508304000 + 0x000..0x158). The island + * is off after a sleep, and the AOP's resume, which reads its ASC first, faults on it. The + * words are read in .begin and put back at the wake, parents first: a word that was on goes + * to a manual target 0xf (in automatic mode it gates again before the AOP's driver gets to + * its ASC), any other gets its saved value if that asks for power. The table + * is the power-down order, children first. + */ +#define T6040_AOP_PS_PA 0x508304000ULL +static const u16 t6040_aop_ps_order[] = { + 0x0b0, 0x128, 0x120, 0x158, 0x150, 0x148, 0x140, 0x138, 0x130, 0x118, 0x0c0, 0x0c8, + 0x0e8, 0x0f0, 0x0f8, 0x100, 0x108, 0x110, 0x118, 0x0e0, 0x0d8, 0x0d0, 0x0b8, 0x090, + 0x020, 0x098, 0x0a0, 0x0a8, 0x028, 0x030, 0x068, 0x070, 0x078, 0x080, 0x038, 0x040, + 0x048, 0x058, 0x060, 0x088, 0x050, 0x010, 0x018, 0x008, 0x000, +}; + +#define T6040_AOP_PS_WORDS 44 + +static void t6040_aop_ps_save(struct t6040_sleep *s) +{ + int i; + + s->aop_ps_saved_valid = false; + if (!s->aop_ps) + return; + for (i = 0; i < T6040_AOP_PS_WORDS; i++) + s->aop_ps_saved[i] = readl(s->aop_ps + PS_STRIDE * i); + s->aop_ps_saved_valid = true; +} + +static void t6040_aop_ps_restore(struct t6040_sleep *s) +{ + int k; + + s->aop_ps_timeouts = 0; + if (!s->aop_ps || !s->aop_ps_saved_valid) + return; + for (k = ARRAY_SIZE(t6040_aop_ps_order) - 1; k >= 0; k--) { + u16 off = t6040_aop_ps_order[k]; + void __iomem *r = s->aop_ps + off; + u32 v, want = s->aop_ps_saved[off / 8], now = readl(r); + + if (FIELD_GET(PS_ACTUAL, want)) { + if (FIELD_GET(PS_ACTUAL, now) == PS_ACTIVE && !(now & PS_AUTO_ENABLE)) + continue; + writel((now & ~PS_AUTO_ENABLE) | PS_TARGET, r); + } else { + if (FIELD_GET(PS_ACTUAL, now) || !FIELD_GET(PS_TARGET, want)) + continue; + writel(want, r); + } + if (readl_poll_timeout_atomic(r, v, FIELD_GET(PS_ACTUAL, v), 1, 2000)) + s->aop_ps_timeouts++; + } +} + +/* + * The power loss resets every power word, and only the domains a Linux driver owns are + * raised again when their devices resume. Here the SIO peripheral words (I2C0 .. UART6 in + * SOC-East) that were fully on at the commit + * and came back lower are raised again, parents first. On J616s those are what the boot + * loader left on and no driver claims: I2C0, SPI0/1/3/4/5 (SPI4 goes to the internal panel), + * the UARTs, AES. Nothing else: every other word is its owner's to raise. + */ +static void t6040_sio_ps_save(struct t6040_sleep *s) +{ + unsigned int i; + + s->sio_ps_saved_valid = false; + for (i = 0; i < ARRAY_SIZE(s->sio_ps_saved); i++) + s->sio_ps_saved[i] = readl(s->soc_east + PS_SIO_FIRST + i * PS_STRIDE); + s->sio_ps_saved_valid = true; +} + +static void t6040_sio_ps_restore(struct t6040_sleep *s) +{ + unsigned int i, n, raised = 0, failed = 0; + u32 val, cur = 0; + + if (!s->sio_ps_saved_valid) + return; + s->sio_ps_saved_valid = false; + for (i = 0; i < ARRAY_SIZE(s->sio_ps_saved); i++) { + u32 off = PS_SIO_FIRST + i * PS_STRIDE; + void __iomem *w = s->soc_east + off; + u32 saved = s->sio_ps_saved[i]; + + if (FIELD_GET(PS_TARGET, saved) != PS_ACTIVE || + FIELD_GET(PS_ACTUAL, saved) != PS_ACTIVE) + continue; + val = readl(w); + if (FIELD_GET(PS_TARGET, val) == PS_ACTIVE) + continue; + writel(val | PS_TARGET, w); + /* Timekeeping is still suspended here: count the wait, no ktime. */ + for (n = 0; n < 1000; n++) { + cur = readl(w); + if (FIELD_GET(PS_ACTUAL, cur) == PS_ACTIVE) + break; + udelay(10); + } + if (n == 1000) + failed++; + raised++; + dev_dbg(s->dev, "wake: SOC-East power word +%#x: %#x at the commit, %#x after the wake, now %#x\n", + off, saved, val, cur); + } + if (failed) + dev_warn(s->dev, "wake: %u of %u SIO power word(s) raised again did not come on\n", + failed, raised); +} + +/* + * The 4 clk-mux park words (ClkGen SOC + these offsets), from + * the clock quiesce; the offsets skip 0x58/0x5c. + */ +static const u32 t6040_clkgen_mux[CLKGEN_MUX_WORDS] = { + 0x38060, 0x38064, 0x38054, 0x38050, +}; + +/* + * The 9 AOP-global wake-source masks armed before an S2R + * (the values are decoded from the ADT's ap-wake-sources). + * valid & ~enabled per word. + */ +static const u32 t6040_wake_mask[AOP_WAKE_WORDS] = { + 0xe5ff5fff, 0xff42edff, 0xffffffff, 0x3ffffbdf, 0xf7ffffc0, + 0xffffff7f, 0xffffffff, 0xffffffff, 0x00003fff, +}; + +/* + * The hardware S2R-lo entry counter: bits 31:3 of MiniPMGR + 0x70970. It tells "the SoC + * entered S2R" from "the engine never finished". Read-only, with the AOP-global S2R + * control, wake masks and wake status, at debug level: at probe, for the boot after a + * failed sleep, and in a dry run. Mapped once at probe: the dry run reads it with + * interrupts off. + */ +#define T6040_S2R_LO_COUNT_PA 0x508370970ULL + +static void t6040_sleep_log_s2r_state(struct t6040_sleep *s, const char *when) +{ + u32 raw = s->s2r_lo_count ? readl(s->s2r_lo_count) : 0; + u32 m[AOP_WAKE_WORDS] = {}, st[AOP_WAKE_WORDS] = {}, ctl = 0; + int i; + + ctl = readl(s->aop_global + AOP_S2R_CTRL); + for (i = 0; i < AOP_WAKE_WORDS; i++) { + m[i] = readl(s->aop_global + AOP_WAKE_MASK(i)); + st[i] = readl(s->aop_global + AOP_WAKE_STATUS(i)); + } + dev_dbg(s->dev, "S2R state at %s: S2R-lo count %u (raw %#x), AOP-global ctl %#x\n", + when, raw >> 3, raw, ctl); + dev_dbg(s->dev, "S2R state at %s: wake masks %*ph\n", when, + (int)sizeof(m), m); + dev_dbg(s->dev, "S2R state at %s: wake status %*ph\n", when, + (int)sizeof(st), st); +} + +/* + * The 15 domains with an axi2af fabric bridge to re-arm. Each domain's power word is in + * SOC (soc_ps, base 0x502800000) or SOC-East (soc_east, base 0x502280000); its + * fabric bridge is a separate window. On wake every domain that is powered up + * (PS TARGET == 0xf) gets bridge +0x104/+0x108 bit0 set, or the bridge drops + * its traffic. + */ +static const struct t6040_axi2af_domain { + const char *name; + bool east; /* power word in soc_east (true) or soc_ps (false) */ + u32 ps_off; + phys_addr_t bridge_pa; +} t6040_axi2af[] = { + [AXI2AF_ATC0_COMMON] = { "atc0_common", false, 0x390, 0x380000000 }, + [AXI2AF_ATC1_COMMON] = { "atc1_common", false, 0x3b8, 0x388000000 }, + [AXI2AF_ATC2_COMMON] = { "atc2_common", false, 0x3e0, 0x390000000 }, + [AXI2AF_ATC3_COMMON] = { "atc3_common", false, 0x408, 0x398000000 }, + [AXI2AF_DISP_SYS] = { "disp_sys", true, 0x258, 0x403000000 }, + [AXI2AF_DISPEXT0_SYS] = { "dispext0_sys", false, 0x340, 0x3a3000000 }, + [AXI2AF_DISPEXT1_SYS] = { "dispext1_sys", false, 0x490, 0x603000000 }, + [AXI2AF_DISPEXT2_SYS] = { "dispext2_sys", false, 0x4e0, 0x607000000 }, + [AXI2AF_DISPEXT3_SYS] = { "dispext3_sys", false, 0x2d8, 0x307000000 }, + [AXI2AF_ANS] = { "ans", true, 0x370, 0x40b000000 }, + [AXI2AF_AVD_SYS] = { "avd_sys", false, 0x170, 0x280000000 }, + [AXI2AF_VENC0_SYS] = { "venc0_sys", false, 0x278, 0x282000000 }, + [AXI2AF_VENC1_SYS] = { "venc1_sys", false, 0x2a0, 0x284000000 }, + [AXI2AF_JPG] = { "jpg", true, 0x3a0, 0x486000000 }, + [AXI2AF_APCIE_SYS_GP] = { "apcie_sys_gp", true, 0x390, 0x416000000 }, +}; + +static void t6040_pmu_write(struct t6040_sleep *s, u16 reg, u8 val) +{ + apple_sleep_pmu_write(s->spmi0, PMU_USID, reg, val); +} + +/* + * Between the steps of the wake return: every access of a step is complete, and a fault it + * raised is taken, before the next step starts. + */ +static void t6040_wake_sync(void) +{ + dsb(sy); + isb(); +} + +/* + * The five complexes off, measured on the J616s: each slot's low nibble to 0, in order + * +0x0 .. +0x10, and the two display complexes (+0xc, +0x10) polled until BIT4 clears. + * The CPU complexes (E, P0, P1) go down at the WFI. + */ +static void t6040_clusters_off(struct t6040_sleep *s) +{ + u32 val; + int i; + + for (i = 0; i < CLUSTER_CTL_NR; i++) { + val = readl(s->cluster_ctl + CLUSTER_CTL(i)); + s->cluster_saved[i] = val; + writel(val & ~CLUSTER_CTL_NIBBLE, s->cluster_ctl + CLUSTER_CTL(i)); + if (i >= CLUSTER_CTL_FIRST_WAIT) + readl_poll_timeout_atomic(s->cluster_ctl + CLUSTER_CTL(i), val, + !(val & CLUSTER_CTL_BUSY), 1, CLUSTER_TIMEOUT_US); + } +} + +static void t6040_east_clk_quiesce(struct t6040_sleep *s, u32 off) +{ + u32 val = readl(s->soc_east + off), cur; + + writel(val & ~SOC_EAST_CLK_AUTO, s->soc_east + off); + readl_poll_timeout_atomic(s->soc_east + off, cur, + FIELD_GET(PS_ACTUAL, cur) == FIELD_GET(PS_TARGET, val), 1, + CLK_QUIESCE_TIMEOUT_US); +} + +/* The clock quiesce, saving the clk-mux words for wake. */ +static void t6040_clkgen_park(struct t6040_sleep *s) +{ + u32 cur; + int i; + + t6040_east_clk_quiesce(s, SOC_EAST_CLK_A); + t6040_east_clk_quiesce(s, SOC_EAST_CLK_B); + for (i = 0; i < CLKGEN_MUX_WORDS; i++) { + void __iomem *reg = s->clkgen + t6040_clkgen_mux[i]; + u32 val = readl(reg); + + s->clkgen_mux_saved[i] = val; + writel((val & CLKGEN_MUX_MASK) | CLKGEN_MUX_SET, reg); + readl_poll_timeout_atomic(reg, cur, !(cur & CLKGEN_MUX_PENDING), 1, + CLK_QUIESCE_TIMEOUT_US); + } + s->clkgen_saved = true; +} + +/* The mirror on wake: the saved clk-mux words back, last parked first. */ +static void t6040_clkgen_restore(struct t6040_sleep *s) +{ + int i; + + if (!s->clkgen_saved) + return; + for (i = CLKGEN_MUX_WORDS - 1; i >= 0; i--) + writel(s->clkgen_mux_saved[i], s->clkgen + t6040_clkgen_mux[i]); + s->clkgen_saved = false; +} + +/* + * Wait for every other core's power word (SOC + 8 * its cpu-start bit index, + * ECPU_0.. PCPU1_5 in the ADT) to read ACTUAL 0, each for at most 10 ms. + */ +static void t6040_secondaries_off(struct t6040_sleep *s) +{ + u32 self = t6040_cpu_start_bit(read_cpuid_mpidr() & MPIDR_HWID_BITMASK); + unsigned int cpu; + u32 v; + + for_each_present_cpu(cpu) { + u32 bit = t6040_cpu_start_bit(cpu_logical_map(cpu)); + + if (!bit || bit == self) + continue; + readl_poll_timeout_atomic(s->soc_ps + PS_STRIDE * __ffs(bit), v, + !FIELD_GET(PS_ACTUAL, v), 1, 10000); + } +} + +/* The hardware steps of the generic PMGR quiesce, in order (see t6040_acc_apsc_pa). */ +static void t6040_pmgr_quiesce(struct t6040_sleep *s) +{ + u64 q; + u32 v; + int i; + + for (i = 0; i < ARRAY_SIZE(t6040_acc_apsc_pa); i++) { + void __iomem *acc = s->acc_apsc[i]; + + if (!acc) + continue; + writeq(readq(acc) | ACC_APSC_DISABLE, acc); + readq_poll_timeout_atomic(acc, q, !(q & ACC_APSC_BUSY), 1, QUIESCE_TIMEOUT_US); + } + for (i = 0; i < ACC_PLL_RELOCK_COMPLEXES; i++) + if (s->acc_apsc[i]) + writeq(readq(s->acc_apsc[i]) & ~ACC_PLL_RELOCK, s->acc_apsc[i]); + for (i = 0; i < ARRAY_SIZE(t6040_tvm); i++) { + if (!s->tvm_ctl[i] || !s->tvm_status[i]) + continue; + writel(readl(s->tvm_ctl[i]) & ~TVM_ENABLE, s->tvm_ctl[i]); + readl_poll_timeout_atomic(s->tvm_status[i], v, !(v & TVM_BUSY), 1, + QUIESCE_TIMEOUT_US); + } +} + +/* The AGX residency gate of the PMGR quiesce (see T6040_AGX_GATE_PA). */ +static void t6040_agx_gate(struct t6040_sleep *s) +{ + u32 gate, val, cur; + + if (!s->agx_gate || !s->afr_ps) + return; + gate = readl(s->agx_gate); + if (!(~gate & AGX_GATE_RESIDENT)) + return; + val = readl(s->afr_ps); + if (!FIELD_GET(PS_TARGET, val)) + return; + writel((val & ~PS_MIN) | FIELD_PREP(PS_MIN, 4), s->afr_ps); + if (readl_poll_timeout_atomic(s->afr_ps, cur, FIELD_GET(PS_MIN, cur) == 4, 1, + AGX_TIMEOUT_US)) + return; + if (readl_poll_timeout_atomic(s->afr_ps, cur, FIELD_GET(PS_ACTUAL, cur) == 4, 1, + AGX_TIMEOUT_US)) + return; + writel(0, s->afr_ps); + readl_poll_timeout_atomic(s->afr_ps, cur, !FIELD_GET(PS_ACTUAL, cur), 1, AGX_TIMEOUT_US); +} + +/* + * Keep-alive: BIT0 of the 64 keep-alive words, PMGR SOC then SOC-East per + * index. Clears them all (on == false) or sets those in @mask; returns the + * mask of the bits that were set (SOC in the low half, SOC-East in the high). + */ +static u64 t6040_keep_alive(struct t6040_sleep *s, bool on, u64 mask) +{ + u64 was = 0; + int i; + + if (!s->ka_soc || !s->ka_east) + return 0; + for (i = 0; i < KA_WORDS; i++) { + void __iomem *a = s->ka_soc + i * KA_STRIDE; + void __iomem *b = s->ka_east + i * KA_STRIDE; + u32 va = readl(a), vb = readl(b); + + was |= (u64)(va & KA_ENABLE) << i | (u64)(vb & KA_ENABLE) << (i + KA_WORDS); + writel((va & ~KA_ENABLE) | (on && (mask & BIT_ULL(i))), a); + writel((vb & ~KA_ENABLE) | (on && (mask & BIT_ULL(i + KA_WORDS))), b); + } + return was; +} + +/* A domain's power word, from whichever window holds it. */ +static void __iomem *t6040_axi2af_ps(struct t6040_sleep *s, const struct t6040_axi2af_domain *d) +{ + void __iomem *base = d->east ? s->soc_east : s->soc_ps; + + return base ? base + d->ps_off : NULL; +} + +/* The preflight's list of the axi2af domains' power words. */ +static void t6040_axi2af_show(struct t6040_sleep *s, struct seq_file *m) +{ + int i; + + seq_puts(m, "axi2af domains (the quiesce gate wants every one off):\n"); + for (i = 0; i < T6040_AXI2AF_DOMAINS; i++) { + void __iomem *ps = t6040_axi2af_ps(s, &t6040_axi2af[i]); + u32 val = ps ? readl(ps) : 0; + + seq_printf(m, " %-13s %#010x%s\n", t6040_axi2af[i].name, val, + ps ? (FIELD_GET(PS_ACTUAL, val) ? " ON" : "") : " (unmapped)"); + } +} + +static int t6040_sleep_preflight_show(struct seq_file *m, void *unused) +{ + struct t6040_sleep *s = m->private; + unsigned int live = t6040_sleep_live_blockers(s, m, NULL); + static const char * const nm[] = { + "EACC", "PACC0", "PACC1_ANE", "DISP_INT", "DISP_EXT", + }; + unsigned int i; + + apple_sleep_registry_show(m); + t6040_axi2af_show(s, m); + seq_printf(m, "verdict: %u live blocker(s)\n", live); + seq_printf(m, "dry_run %u, wd2_secs %u, allow_entry %d, parking ops %s\n", + READ_ONCE(dry_run), READ_ONCE(wd2_secs), allow_entry, + s->power_ops ? "registered" : "not registered"); + for (i = 0; i < CLUSTER_CTL_NR; i++) + seq_printf(m, "cluster-ctl +%#x %-9s %#x\n", CLUSTER_CTL(i), nm[i], + readl(s->cluster_ctl + CLUSTER_CTL(i))); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(t6040_sleep_preflight); + +/* + * Read-only axi2af gate: every reachable domain's power word is expected to read + * [7:4] == 0 here. The ones still on are only named, at debug level. + */ +static void t6040_axi2af_readcheck(struct t6040_sleep *s) +{ + int i, on = 0; + + for (i = 0; i < T6040_AXI2AF_DOMAINS; i++) { + void __iomem *ps = t6040_axi2af_ps(s, &t6040_axi2af[i]); + u32 val; + + if (!ps) + continue; + val = readl(ps); + if (FIELD_GET(PS_ACTUAL, val)) { + dev_dbg(s->dev, "axi2af %s still on (%#x) at commit\n", + t6040_axi2af[i].name, val); + on++; + } + } + if (on) + dev_dbg(s->dev, "%d axi2af domain(s) still powered at commit\n", on); +} + +/* + * The wake-side axi2af re-arm: every retained domain powered back up (PS TARGET 0xf and + * ACTUAL on) gets its fabric bridge +0x104/+0x108 bit0 set, or the bridge drops its + * traffic. A word in automatic mode can read TARGET 0xf with its domain gated; the bridge + * is then off too, and an access to it raises an SError, so such a domain is left alone. + */ +static void t6040_axi2af_rearm(struct t6040_sleep *s) +{ + int i; + + for (i = 0; i < T6040_AXI2AF_DOMAINS; i++) { + void __iomem *ps = t6040_axi2af_ps(s, &t6040_axi2af[i]); + void __iomem *bridge = s->axi2af_bridge[i]; + u32 val; + + if (!ps || !bridge) + continue; + val = readl(ps); + if (FIELD_GET(PS_TARGET, val) != PS_ACTIVE) + continue; + if (!FIELD_GET(PS_ACTUAL, val)) { + dev_dbg(s->dev, "wake: axi2af %s not re-armed, PS %#x (gated)\n", + t6040_axi2af[i].name, val); + continue; + } + t6040_wake_sync(); + writel(readl(bridge + AXI2AF_BRIDGE_104) | BIT(0), bridge + AXI2AF_BRIDGE_104); + writel(readl(bridge + AXI2AF_BRIDGE_108) | BIT(0), bridge + AXI2AF_BRIDGE_108); + t6040_wake_sync(); + } +} + +/* Arm the 9 AOP-global wake masks before the commit, saving the old words. */ +static void t6040_wake_arm(struct t6040_sleep *s) +{ + int i; + + for (i = 0; i < AOP_WAKE_WORDS; i++) { + s->wake_mask_saved[i] = readl(s->aop_global + AOP_WAKE_MASK(i)); + writel(t6040_wake_mask[i], s->aop_global + AOP_WAKE_MASK(i)); + } + s->wake_mask_saved_valid = true; +} + +/* Put the wake masks back at .end of a returned (or refused) sleep. */ +static void t6040_wake_restore(struct t6040_sleep *s) +{ + int i; + + if (!s->wake_mask_saved_valid) + return; + for (i = 0; i < AOP_WAKE_WORDS; i++) + writel(s->wake_mask_saved[i], s->aop_global + AOP_WAKE_MASK(i)); + s->wake_mask_saved_valid = false; +} + +/* + * One power word raised, as sleep.c's sleep_ps_raise() does: target 0xf with automatic + * mode off and the flag bits cleared, ACTUAL polled, then AUTO_ENABLE. The two halves + * are separate for a word that has to stay at a manual target for a while or for good. + */ +static int t6040_ps_raise_start(void __iomem *base, u32 offset) +{ + return apple_sleep_ps_raise_start(base + offset); +} + +static void t6040_ps_raise_finish(void __iomem *base, u32 offset) +{ + apple_sleep_ps_raise_finish(base + offset); +} + +static int t6040_ps_raise(void __iomem *base, u32 offset) +{ + int ret = t6040_ps_raise_start(base, offset); + + t6040_ps_raise_finish(base, offset); + + return ret; +} + +static bool t6040_ps_actual_on(void __iomem *base, u32 offset) +{ + return FIELD_GET(PS_ACTUAL, readl(base + offset)); +} + +/* The shared PHY and PHY-IP to sleep: the PCIe root complex disable sequence. */ +static void t6040_apcie_phy_off(struct t6040_sleep *s) +{ + struct t6040_apcie_st *a = &s->apcie_st; + void __iomem *ctrl = a->phy + APCIE_PHY_CTRL; + void __iomem *run = a->phy + APCIE_PHY_RUN; + u32 val; + + writel(readl(ctrl) | APCIE_PHY_CTRL_UP, ctrl); + udelay(10); + writel(readl(a->phy_ip + 0x38) & ~GENMASK(27, 26), a->phy_ip + 0x38); + writel(readl(a->phy_ip + 0x8c) | GENMASK(27, 24), a->phy_ip + 0x8c); + udelay(5); + writel(readl(ctrl) | APCIE_PHY_CTRL_RESET, ctrl); + udelay(1); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK1REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK1ACK), 1, APCIE_POLL_US); + udelay(1); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_CLK0REQ, ctrl); + readl_poll_timeout_atomic(ctrl, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_POLL_US); + udelay(1); + writel(readl(ctrl) & ~APCIE_PHY_CTRL_UP, ctrl); + udelay(10); + writel(readl(run) & ~APCIE_PHY_RUN_EN2, run); + writel(readl(run) & ~APCIE_PHY_RUN_EN, run); +} + +/* APCIE_PHY_SW to clock-gated, its busy bit polled clear. */ +static void t6040_apcie_phy_sw_gate(struct t6040_sleep *s) +{ + void __iomem *reg = s->soc_east + PS_APCIE_PHY_SW; + u32 val; + + writel(PS_APCIE_PHY_SW_CLKGATED, reg); + readl_poll_timeout_atomic(reg, val, !(val & PS_BUSY), 1, PS_OFF_TIMEOUT_US); +} + +/* The PCIe PLL shutdown: the clock generator's control word. */ +static void t6040_apcie_pll_off(struct t6040_sleep *s) +{ + void __iomem *clk = s->apcie_st.clkgen + APCIE_CLK_CTRL; + u32 val; + + writel((readl(clk) & ~(APCIE_CLK_RUN | GENMASK(4, 3))) | GENMASK(4, 3), clk); + writel(readl(clk) | APCIE_CLK_OFF, clk); + writel(readl(clk) & ~BIT(0), clk); + readl_poll_timeout_atomic(clk, val, !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + writel(readl(clk) | APCIE_CLK_OFF2, clk); +} + +/* + * The sleep engine hands each ATC port to its always-on side and then powers the port's + * common domain off itself: it needs the four ATC commons on at that + * point. With them off it waits for good raising ATC0_COMMON, whose parent FAB2_SOC is + * off as well. FAB2_SOC is raised by hand, then ATC0..3_COMMON to a manual target 0xf, + * then FAB2_SOC gets its automatic mode back: the powered children hold it up. + */ +static const u16 t6040_atc_common[4] = { 0x390, 0x3b8, 0x3e0, 0x408 }; + +static void t6040_atc_commons_on(struct t6040_sleep *s) +{ + void __iomem *b = s->soc_ps; + int i; + + t6040_ps_raise_start(b, PS_FAB2_SOC); + for (i = 0; i < ARRAY_SIZE(t6040_atc_common); i++) + t6040_ps_raise_start(b, t6040_atc_common[i]); + t6040_ps_raise_finish(b, PS_FAB2_SOC); +} + +/* + * The CFG writes of the PHY sleep entry, run on every ATC port before a sleep: + * ATCPHY_CFG +0x2c bit 0 (S2R_AON_SYNC_RST_N), +0x08 bit 0 (CFG_S2R_FSM_EN) and +0x28 bit 0 + * (AHB_CLK_REQ_N) set, then +0x04 bit 8 (the clock's ack) polled. Without them the engine + * waits for good on a port that carries DisplayPort lanes: port 3 (the HDMI output) always, + * any other port once a display runs on it in DP alternate mode. A parked port, and + * one its PHY driver has put to sleep, already has +0x28 bit 0 set and is left alone. + */ +#define ATC_CFG_CLK_STATUS 0x04 +#define ATC_CFG_CLK_ACK BIT(8) +#define ATC_CFG_S2R_FSM 0x08 +#define ATC_CFG_S2R_FSM_EN BIT(0) +#define ATC_CFG_AHB_CLK 0x28 +#define ATC_CFG_AHB_CLK_REQ_N BIT(0) +#define ATC_CFG_S2R_AON 0x2c +#define ATC_CFG_S2R_AON_SYNC_RST_N BIT(0) + +static void t6040_atc_s2r_entry(struct t6040_sleep *s) +{ + u32 v; + int n; + + for (n = 0; n < ARRAY_SIZE(s->atc_cfg); n++) { + void __iomem *c = s->atc_cfg[n]; + + if (!c || (readl(c + ATC_CFG_AHB_CLK) & ATC_CFG_AHB_CLK_REQ_N)) + continue; + writel(readl(c + ATC_CFG_S2R_AON) | ATC_CFG_S2R_AON_SYNC_RST_N, + c + ATC_CFG_S2R_AON); + udelay(1); + writel(readl(c + ATC_CFG_S2R_FSM) | ATC_CFG_S2R_FSM_EN, c + ATC_CFG_S2R_FSM); + udelay(5); + writel(readl(c + ATC_CFG_AHB_CLK) | ATC_CFG_AHB_CLK_REQ_N, c + ATC_CFG_AHB_CLK); + readl_poll_timeout_atomic(c + ATC_CFG_CLK_STATUS, v, v & ATC_CFG_CLK_ACK, 1, 1000); + } +} + +/* + * The 13 fabric power words. The fabrics are expected up through the entry, and the wake + * polls these words for automatic mode with target 0xf. Under Linux genpd has powered some + * of them off with their last consumer by now (FAB2_SOC with the ATC commons at + * suspend_noirq). Right before the S2R timer each one not in that state is raised: target + * 0xf, ACTUAL 0xf polled, then automatic mode. The ATC commons and the ports' PHY sleep + * entry follow, in this order. + */ +static const struct { + bool east; + u16 off; +} t6040_fabrics[13] = { + { false, 0x110 } /* AFI */, { false, 0x178 } /* AFC */, + { false, 0x448 } /* SBR */, { false, 0x100 } /* PMS */, + { false, 0x168 } /* FAB0_SOC */, { false, 0x2d0 } /* FAB1_SOC */, + { false, 0x338 } /* FAB2_SOC */, { false, 0x440 } /* FAB5_SOC */, + { false, 0x480 } /* FAB6_SOC */, { false, 0x488 } /* FAB7_SOC */, + { true, 0x118 } /* FAB3_SOC */, { true, 0x128 } /* FAB4_SOC */, + { false, 0x298 } /* FAB_AFR */, +}; + +static void t6040_fabrics_on(struct t6040_sleep *s) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(t6040_fabrics); i++) { + void __iomem *b = t6040_fabrics[i].east ? s->soc_east : s->soc_ps; + u32 off = t6040_fabrics[i].off, v; + + v = readl(b + off); + if (!(v & PS_AUTO_ENABLE) || FIELD_GET(PS_TARGET, v) != PS_ACTIVE) + t6040_ps_raise(b, off); + } + t6040_atc_commons_on(s); + t6040_atc_s2r_entry(s); +} + +/* + * The memory channels' power gating. The sleep engine does not write the DCS power words: it + * expects each DCS to gate itself after self-refresh. That takes the DCS's dynamic power-gate + * mode (DCS +0x20 bits 12 and 8) and its power word in automatic mode with no minimum state. + * Both are expected at boot and again after each wake. Nothing sets them under Linux (the DT leaves + * the ps_dcs_* domains disabled: target 0xf, no automatic mode), and the engine then waits + * for the first DCS pair's power gate for good. Both are applied in each .begin. + */ +static const struct { + bool east; + u16 off; +} t6040_dcs_ps[16] = { + { false, 0x188 }, { false, 0x1a0 }, { false, 0x1b0 }, { false, 0x1c0 }, + { true, 0x2e0 }, { true, 0x2e8 }, { true, 0x2f0 }, { true, 0x2f8 }, + { false, 0x1d0 }, { false, 0x1e0 }, { false, 0x1f0 }, { false, 0x200 }, + { true, 0x300 }, { true, 0x308 }, { true, 0x310 }, { true, 0x318 }, +}; + +static void __iomem *t6040_dcs_ps_reg(struct t6040_sleep *s, int i) +{ + void __iomem *b = t6040_dcs_ps[i].east ? s->soc_east : s->soc_ps; + + return b + t6040_dcs_ps[i].off; +} + +/* + * In order: the minimum state cleared, target 0xf, ACTUAL 0xf polled, then automatic + * mode. Returns a mask of the DCS whose ACTUAL did not reach 0xf. + */ +static u32 t6040_dcs_ps_auto_on(struct t6040_sleep *s) +{ + u32 to = 0, v; + int i; + + for (i = 0; i < ARRAY_SIZE(t6040_dcs_ps); i++) { + void __iomem *r = t6040_dcs_ps_reg(s, i); + + v = readl(r); + if ((v & PS_AUTO_ENABLE) && !(v & PS_MIN)) + continue; + writel((v & ~(PS_MIN | PS_FLAGS | PS_TARGET)) | + FIELD_PREP(PS_TARGET, PS_ACTIVE), r); + if (readl_poll_timeout_atomic(r, v, FIELD_GET(PS_ACTUAL, v) == PS_ACTIVE, 1, + PS_OFF_TIMEOUT_US)) + to |= BIT(i); + writel(readl(r) | PS_AUTO_ENABLE, r); + } + return to; +} + +#define DCS_PG_CTL 0x20 +#define DCS_PG_DYN (BIT(12) | BIT(8)) + +static void t6040_dcs_dyn_pg(struct t6040_sleep *s) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(s->dcs); i++) { + u32 v; + + if (!s->dcs[i]) + continue; + v = readl(s->dcs[i] + DCS_PG_CTL); + if ((v & DCS_PG_DYN) != DCS_PG_DYN) + writel(v | DCS_PG_DYN, s->dcs[i] + DCS_PG_CTL); + } +} + +/* + * ANS's own fabric bridge (0x4_0b00_0000 +0x104/+0x108 bit 0). The bit is cleared before a + * retained domain goes off and set again once it is back up; here the wake sets it + * (t6040_apcie_st_raise()) and the SoC's sleep engine, not this driver, takes ANS down. + * The boot loader leaves the bit clear, so a boot's first sleep goes down with it clear; + * without this the next one meets the bridge as the first wake armed it. Only while ANS is + * on: the bridge sits in it. + */ +static void t6040_ans_bridge_quiesce(struct t6040_sleep *s) +{ + void __iomem *bridge = s->axi2af_bridge[AXI2AF_ANS]; + u32 a, b; + + if (!bridge || + FIELD_GET(PS_ACTUAL, readl(s->soc_east + PS_ANS)) != PS_ACTIVE) + return; + a = readl(bridge + AXI2AF_BRIDGE_104); + b = readl(bridge + AXI2AF_BRIDGE_108); + if (!((a | b) & BIT(0))) + return; + writel(a & ~BIT(0), bridge + AXI2AF_BRIDGE_104); + writel(b & ~BIT(0), bridge + AXI2AF_BRIDGE_108); +} + +/* + * Quiesce ANS's storage link, once ANS has been stopped by its driver. The PHY, the + * clock generator and the AXI enables are treated as shared by the two ST lanes. + * + * The power words of this group are left to the SoC's sleep engine, which powers it down + * itself and in its own order: APCIE_PHY_SW to clock-gated, APCIE_SYS_ST0/1 off, APCIE_ST0/1 + * clock-gated, ANS off, APCIE_SYS_GP off, APCIE_GP clock-gated. It polls APCIE_PHY_SW for + * its parents, which are both SYS_ST lanes: with them lowered here, as the T8140 controller + * does, it waits there for good. So the commit raises APCIE_SYS_ST1, which + * Linux never uses, and does only what the engine needs before it runs: the shared PHY put + * to sleep, APCIE_PHY_SW clock-gated while APCIE_GP is still on, the PCIe PLL shut down. + */ +static void t6040_apcie_st_lower(struct t6040_sleep *s) +{ + struct t6040_apcie_st *a = &s->apcie_st; + void __iomem *ps = s->soc_east; + + if (!a->valid || !ps) + return; + + /* The PHY's registers need a lane domain on; quiet it first (shared). */ + if (a->phy && t6040_ps_actual_on(ps, PS_APCIE_SYS_ST0)) + t6040_apcie_phy_off(s); + t6040_apcie_phy_sw_gate(s); + if (a->clkgen && t6040_ps_actual_on(ps, PS_APCIE_SYS_ST0)) + t6040_apcie_pll_off(s); + + /* AXI enables off (shared). */ + if (a->axi) { + writel(readl(a->axi + APCIE_AXI_EN0) & ~BIT(0), a->axi + APCIE_AXI_EN0); + writel(readl(a->axi + APCIE_AXI_EN1) & ~BIT(0), a->axi + APCIE_AXI_EN1); + } + t6040_ans_bridge_quiesce(s); +} + +static void t6040_bridge_rearm(void __iomem *bridge) +{ + if (!bridge) + return; + writel(readl(bridge + AXI2AF_BRIDGE_104) | BIT(0), bridge + AXI2AF_BRIDGE_104); + writel(readl(bridge + AXI2AF_BRIDGE_108) | BIT(0), bridge + AXI2AF_BRIDGE_108); +} + +/* + * Raise ANS's storage link from power-off, the two-lane adaptation of + * sleep.c's apple_sleep_raise_apcie(). Runs atomic on the waking core in + * t6040_sleep_return(), after t6040_axi2af_rearm() and before control returns to + * the PM core, so nvme's resume_early sets the ANS run bit onto a live link. + * Both lanes come up; the PLL/clkgen/PHY tables run once (shared). The two + * apcie_sys_st domains are not in the 15-domain axi2af table, so their bridges + * get no re-arm. + * + * The AXI enables (+0x104/+0x108 of 0x4_1600_0000) are the fabric bridge of + * APCIE_SYS_GP, which the commit lowered together with APCIE_GP: both are raised + * here before the enables are written. APCIE_PHY_SW, which the commit gated, + * comes back before the PHY, and the PHY's reference PLL (common +0 bit 31) is + * waited for before the PHY's clocks are requested. @apcie_wake_step tells a + * failed raise's stage. + */ +static int t6040_apcie_st_raise(struct t6040_sleep *s) +{ + struct t6040_apcie_st *a = &s->apcie_st; + void __iomem *ps = s->soc_east; + void __iomem *ctrl, *run; + u32 val; + int ret; + + if (!a->valid || !ps) + return 0; + ctrl = a->phy + APCIE_PHY_CTRL; + run = a->phy + APCIE_PHY_RUN; + + /* + * In order, each domain to ACTUAL 0xf: ANS, GP, SYS_GP, ST0, SYS_ST0, ST1, SYS_ST1, + * the fabric-bridge bit0s set after ANS and after SYS_GP. SYS_GP stays at a manual + * target 0xf: in automatic mode it gates again, and the DART behind it then faults in + * its resume. + */ + ret = t6040_ps_raise(ps, PS_ANS); + if (!ret) + t6040_bridge_rearm(s->axi2af_bridge[AXI2AF_ANS]); + ret |= t6040_ps_raise(ps, PS_APCIE_GP); + if (!ret) + ret = t6040_ps_raise_start(ps, PS_APCIE_SYS_GP); + if (!ret) { + writel(readl(a->axi + APCIE_AXI_EN0) | BIT(0), a->axi + APCIE_AXI_EN0); + writel(readl(a->axi + APCIE_AXI_EN1) | BIT(0), a->axi + APCIE_AXI_EN1); + } + ret |= t6040_ps_raise(ps, PS_APCIE_ST0); + ret |= t6040_ps_raise(ps, PS_APCIE_SYS_ST0); + ret |= t6040_ps_raise(ps, PS_APCIE_ST1); + ret |= t6040_ps_raise(ps, PS_APCIE_SYS_ST1); + t6040_wake_sync(); + s->apcie_wake_step = 1; + if (ret) + return ret; + + /* Tunables + clkgen (shared). Both OFF bits clear; only a power-off cleared them. */ + apple_tunable_apply(a->axi, a->axi_tunable); + apple_tunable_apply(a->cio3pll, a->cio3pll_tunable); + apple_tunable_apply(a->clkgen, a->clkgen_tunable); + writel(readl(a->clkgen + APCIE_CLK_START) | APCIE_CLK_START_GO, + a->clkgen + APCIE_CLK_START); + writel(readl(a->clkgen + APCIE_CLK_CTRL) & ~(APCIE_CLK_OFF2 | APCIE_CLK_OFF), + a->clkgen + APCIE_CLK_CTRL); + ret = readl_poll_timeout_atomic(a->clkgen + APCIE_CLK_CTRL, val, + !(val & APCIE_CLK_BUSY), 1, APCIE_POLL_US); + t6040_wake_sync(); + s->apcie_wake_step = 2; + if (ret) + return ret; + + ret = t6040_ps_raise(ps, PS_APCIE_PHY_SW); + s->apcie_wake_step = 3; + if (ret) + return ret; + + /* Shared PHY bring-up + PLL/lane tables (as sleep.c's sleep_apcie_phy_ip_apply()). */ + apple_tunable_apply(a->phy, a->phy_tunable); + ret = readl_poll_timeout_atomic(a->phy + APCIE_PHY_COMMON, val, val & BIT(31), 1, + APCIE_POLL_US); + t6040_wake_sync(); + s->apcie_wake_step = 4; + if (ret) + return ret; + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK0REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK0ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) | APCIE_PHY_CTRL_CLK1REQ, ctrl); + ret = readl_poll_timeout_atomic(ctrl, val, val & APCIE_PHY_CTRL_CLK1ACK, 1, APCIE_POLL_US); + if (ret) + return ret; + writel(readl(ctrl) & ~APCIE_PHY_CTRL_RESET, ctrl); + t6040_wake_sync(); + s->apcie_wake_step = 5; + + writel(readl(run) | APCIE_PHY_RUN_EN, run); + apple_sleep_tunable_apply_changed(a->phy_ip, a->phy_ip_pll); + apple_sleep_tunable_apply_changed(a->phy_ip, a->phy_ip_lanes); + writel(readl(a->phy_ip + APCIE_PHY_IP_PLL_CTL) & ~APCIE_PHY_IP_PLL_HOLD, + a->phy_ip + APCIE_PHY_IP_PLL_CTL); + writel(readl(run) | APCIE_PHY_RUN_EN2, run); + ret = readl_poll_timeout_atomic(a->phy + APCIE_PHY_READY, val, val & BIT(0), 1, + APCIE_POLL_US); + if (ret) + return ret; + t6040_wake_sync(); + s->apcie_wake_step = 6; + writel(readl(a->phy + APCIE_PHY_COMMON) | BIT(0), a->phy + APCIE_PHY_COMMON); + writel(readl(ctrl) | APCIE_PHY_CTRL_UP, ctrl); + t6040_wake_sync(); + s->apcie_wake_step = 7; + + /* RC ready handshake (as T8140's). */ + writel(0, a->rc + APCIE_RC_ZERO); + writel(0x140, a->rc + APCIE_RC_READY_CFG); + writel(1, a->rc + APCIE_RC_READY_START); + return readl_poll_timeout_atomic(a->rc + APCIE_RC_READY_STAT, val, val & BIT(0), 1, + APCIE_POLL_US); +} + +/* + * The first thing Linux runs after a wake, MMU off and no stack (as the T8132 stub does). The + * last wake program leaves the AON WD2 running for 5 s, less than the device resume takes: + * restart it for 60 s, in t6040_sleep_set_wd2()'s order, then take cpu_resume. The PA is + * the T6040 AON WDT (T6040_WDT_PA), used only when the DT's wdt block is that one. + */ +asm(" .pushsection .text, \"ax\"\n" +" .type t6040_resume_stub, %function\n" +"t6040_resume_stub:\n" +" movz x9, #0xc000\n" +" movk x9, #0x0836, lsl #16\n" +" movk x9, #0x5, lsl #32\n" +" str wzr, [x9, #0x2c]\n" +" str wzr, [x9, #0x20]\n" +" movz w10, #0xa800\n" +" movk w10, #0x55d4, lsl #16\n" +" str w10, [x9, #0x28]\n" +" mov w10, #1\n" +" str w10, [x9, #0x2c]\n" +" b cpu_resume\n" +" .size t6040_resume_stub, . - t6040_resume_stub\n" +" .popsection\n" +); +extern char t6040_resume_stub[]; + +/* + * The four ATC complexes' power words in SOC (soc_ps, 0x502800000), deepest first: cio_usb, + * cio_pcie, cio, pcie, common (ADT: ATCn_CIO_USB .. ATCn_COMMON). The quiesce gate wants + * them off; their drivers don't lower atcN_common. + * + * Their common_dp, usb, phymxwrap and usb_aon words are MiniPMGR words (0x508300000 + 0xa0.., + * ADT ATCn_COMMON_DP/USB/PHYMXWRAP/USB_AON) and not in this table: the same offsets in SOC + * are AIC, PMC, PMS_FPWM0-4 and FAB0_SOC. + */ +static const u16 t6040_atc_words[4][5] = { + { 0x3b0, 0x3a8, 0x398, 0x3a0, 0x390 }, + { 0x3d8, 0x3d0, 0x3c0, 0x3c8, 0x3b8 }, + { 0x400, 0x3f8, 0x3e8, 0x3f0, 0x3e0 }, + { 0x438, 0x430, 0x420, 0x428, 0x408 }, +}; + +/* A domain lowered (T8140 measured): AUTO_ENABLE off first, then target 0, poll. */ +static int t6040_ps_lower_word(void __iomem *reg) +{ + u32 val = readl(reg); + + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, reg); + } + writel((val & ~PS_TARGET) | PS_FLAGS, reg); + return readl_poll_timeout_atomic(reg, val, !FIELD_GET(PS_ACTUAL, val), 1, + PS_OFF_TIMEOUT_US); +} + +static void t6040_atc_lower(struct t6040_sleep *s) +{ + int p, k; + + for (p = 0; p < ARRAY_SIZE(t6040_atc_words); p++) + for (k = 0; k < ARRAY_SIZE(t6040_atc_words[0]); k++) { + void __iomem *reg = s->soc_ps + t6040_atc_words[p][k]; + u32 val = readl(reg); + + if (!FIELD_GET(PS_TARGET, val) && !FIELD_GET(PS_ACTUAL, val)) + continue; + t6040_ps_lower_word(reg); + } +} + +/* + * T8140's radio port stop, on the same t602x port block: the link into L2, the endpoint's + * PERST# asserted, the PHY's clocks off. The endpoint keeps its power and comes back by its + * host after the sleep. + */ +static void t6040_gp_port_stop(struct t6040_sleep *s, int p) +{ + void __iomem *port = s->gp_port[p]; + void __iomem *phy = s->gp_phy[p]; + u32 val; + + if (!port || !phy) + return; + s->gp_phy_saved[p] = readl(phy); + s->gp_appclk_saved[p] = readl(port + APCIE_PORT_APPCLK); + s->gp_perst_saved[p] = readl(port + APCIE_PORT_PERST); + s->gp_saved[p] = true; + writel(readl(port + APCIE_PORT_INTMSK) | APCIE_PORT_INT_LINK, port + APCIE_PORT_INTMSK); + writel(APCIE_PORT_INT_LINK, port + APCIE_PORT_INTSTAT); + writel(readl(port + APCIE_PORT_APPCLK) | APCIE_PORT_APPCLK_CGDIS, port + APCIE_PORT_APPCLK); + if (readl(port + APCIE_PORT_LINKSTS) & APCIE_PORT_LINKSTS_UP) { + writel(APCIE_PORT_L2_ENTER, port + APCIE_PORT_L2); + readl_poll_timeout_atomic(port + APCIE_PORT_L2, val, !(val & APCIE_PORT_L2_BUSY), + 10, APCIE_PORT_STOP_US); + } + if (s->gp_pins && s->gp_perst[p] >= 0) { + void __iomem *reg = s->gp_pins + 4 * s->gp_perst[p]; + + val = readl(reg); + writel(s->gp_perst_low[p] ? val & ~GP_PIN_DATA : val | GP_PIN_DATA, reg); + } + writel(readl(phy) & ~APCIE_PORT_PHY_LINK, phy); + udelay(1000); + /* What this bit does isn't known. It is set around PERST# (T8140). */ + writel(readl(port + APCIE_PORT_PERST_GUARD) | BIT(8), port + APCIE_PORT_PERST_GUARD); + writel(0, port + APCIE_PORT_PERST); + writel(readl(port + APCIE_PORT_PERST_GUARD) & ~BIT(8), port + APCIE_PORT_PERST_GUARD); + writel(readl(phy) & ~APCIE_PORT_PHY_EN1, phy); + writel(readl(phy) & ~APCIE_PORT_PHY_EN0, phy); + writel(readl(phy) | APCIE_PHY_CTRL_RESET, phy); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK0REQ, phy); + readl_poll_timeout_atomic(phy, val, !(val & APCIE_PHY_CTRL_CLK0ACK), 1, APCIE_PORT_STOP_US); + writel(readl(phy) & ~APCIE_PHY_CTRL_CLK1REQ, phy); + writel(readl(port + APCIE_PORT_APPCLK) & ~APCIE_PORT_APPCLK_EN, port + APCIE_PORT_APPCLK); + writel(readl(port + APCIE_PORT_INTSTAT), port + APCIE_PORT_INTSTAT); +} + +/* Whether the GP host's system domain is still on; if not, there is nothing left to stop. */ +static __always_inline bool t6040_apcie_sys_gp_on(struct t6040_sleep *s) +{ + return FIELD_GET(PS_ACTUAL, readl(s->soc_east + PS_APCIE_SYS_GP)); +} + +/* The GP host's two ports, while the block's domains are all still on. */ +static void t6040_gp_ports_stop(struct t6040_sleep *s) +{ + int p; + + if (!t6040_apcie_sys_gp_on(s)) + return; + for (p = 0; p < ARRAY_SIZE(s->gp_port); p++) + t6040_gp_port_stop(s, p); +} + +/* + * apcie_sys_gp (AUTO off, axi2af bit0s, target 0), then apcie_gp. After ANS's ST lanes: the + * storage lanes' cio3pll/clkgen/PHY windows sit in apcie_gp, and an access to them with it + * off hangs the bus. T8140's order: SYS_ST, ST, SYS_GP, GP. APCIE_GP goes to clock-gated + * (TARGET 4), the state the sleep engine itself asks for, not to off. + */ +static void t6040_apcie_gp_lower(struct t6040_sleep *s) +{ + void __iomem *sys, *gp, *bridge = s->axi2af_bridge[AXI2AF_APCIE_SYS_GP]; + u32 val; + + sys = s->soc_east + PS_APCIE_SYS_GP; + gp = s->soc_east + PS_APCIE_GP; + if (!t6040_apcie_sys_gp_on(s)) + return; + + val = readl(sys); + if (val & PS_AUTO_ENABLE) { + val &= ~(PS_AUTO_ENABLE | PS_FLAGS); + writel(val, sys); + } + if (bridge) { + writel(readl(bridge + AXI2AF_BRIDGE_104) & ~BIT(0), bridge + AXI2AF_BRIDGE_104); + writel(readl(bridge + AXI2AF_BRIDGE_108) & ~BIT(0), bridge + AXI2AF_BRIDGE_108); + } + writel((val & ~PS_TARGET) | PS_FLAGS, sys); + readl_poll_timeout_atomic(sys, val, !FIELD_GET(PS_ACTUAL, val), 1, PS_OFF_TIMEOUT_US); + + writel(PS_APCIE_GP_CLKGATED, gp); + readl_poll_timeout_atomic(gp, val, FIELD_GET(PS_ACTUAL, val) == 4, 1, PS_OFF_TIMEOUT_US); +} + +/* + * The real commit (t6040_sleep_commit, reached through cpu_suspend): t6040_sleep_enter + * calls it only when no blocker remains and allow_entry is set. In order: the stopped CPUs + * released to their last WFI, the ATC complexes and the GP PCIe ports lowered, ANS's storage + * link quiesced, WD0 off and the AON WD2 net armed, the PMU's boot stage 0x50 and + * NOTIFY_ENABLE over spmi/spmi-a0, the steps of the PMGR quiesce (APSC, PLL relock, + * TVMs, the AGX gate, the five complexes, the axi2af read-check, the clock quiesce, the + * keep-alive bits), aura's mailbox armed with the resume entry, the fabrics and the ATC + * ports readied for the sleep engine, the MasterMini S2R timer (0x508904000 := 1), the AIC + * masked and the deep WFI. The watchdog, PMU, cluster, axi2af and timer writes are as + * measured on the J616s. A hang here ends at WD2. + */ +static int t6040_sleep_commit(unsigned long unused) +{ + struct t6040_sleep *s = t6040_sleep; + + s->siq_cfg = read_sysreg_s(SYS_APL_SIQ_CFG_EL1); + spin_table_stop_parked(); + /* + * Every other core has to be off, in its last WFI, before the boot core turns the + * complexes off, or the sleep engine waits for them: release them now, and wait for + * their power words to read off further down. + */ + spin_table_release_stopped(); + t6040_atc_lower(s); + t6040_gp_ports_stop(s); + + /* + * ANS's storage link, once ANS has been stopped by its driver (see + * t6040_apcie_phy_off). It follows every refusal, so a refused sleep finds + * the link up. + */ + t6040_ps_raise_start(s->soc_east, PS_APCIE_SYS_ST1); + if (s->pcie_ans_stopped) + t6040_apcie_st_lower(s); + t6040_apcie_gp_lower(s); + + /* WD0 off, then the AON WD2 net (5 s covers the reconfig). */ + writel(0, s->wdt + WDT_WD0_CTL); + t6040_sleep_set_wd2(s, clamp(READ_ONCE(sleep_wd_secs), 1U, WDT_MAX_SECS)); + /* And the SMC's AP watchdog deadline off, right after the WD2 (measured). */ + if (s->smc_wdt_deadline) + writel(0, s->smc_wdt_deadline); + + /* PMU boot stage 0x50 and NOTIFY_ENABLE on both SPMI controllers (measured). */ + t6040_pmu_write(s, PMU_BOOT_STAGE, PMU_BOOT_STAGE_SLEEP); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi0 + SPMI_NOTIFY_ENABLE); + writel(SPMI_NOTIFY_ENABLE_SLEEP, s->spmi_a0 + SPMI_NOTIFY_ENABLE); + + /* + * The order of the PMGR quiesce (the cluster and axi2af + * parts measured on the J616s): the five complexes off, the axi2af gate read, + * the clock quiesce, the keep-alive bits off, then the S2R timer. + */ + t6040_secondaries_off(s); + t6040_pmgr_quiesce(s); + t6040_agx_gate(s); + t6040_clusters_off(s); + + /* Read-only: the axi2af panic gate checked before the engine goes. */ + t6040_axi2af_readcheck(s); + t6040_clkgen_park(s); + s->ka_on_before = t6040_keep_alive(s, false, 0); + + /* Arm aura's AURAS2R1 mailbox with this boot's resume entry. */ + if (s->mbox) { + WRITE_ONCE(s->mbox->resume_pa, + s->wdt_stub_ok ? + __pa_symbol(t6040_resume_stub) : __pa_symbol(cpu_resume)); + dcache_clean_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + dsb(sy); + } + + t6040_fabrics_on(s); + t6040_sio_ps_save(s); + + /* The commit: the MasterMini S2R timer arms the SoC sleep engine. */ + writel(1, s->mastermini + MASTERMINI_S2R_TIMER); + dsb(sy); + + /* A raised line would end the WFI the engine waits for. */ + aic_sleep_mask_all(); + spin_table_release_stopped(); + + t6040_deep_wfi(); +} + +/* + * The GP PCIe host's two ports (Wi-Fi/BT, SD): the boot loader set their clocks up, and + * nobody does after a sleep. Without them the DART in port 0's block faults in its noirq + * resume. t6040_gp_port_stop() backwards, short of the link: the lane reference clocks + * requested and acknowledged, the PHY's enables and the port's application clock as saved, + * the controller's PERST register as saved, then the port's ready bit polled. The link + * itself stays down for the host driver. + */ +static void t6040_gp_pcie_clk(struct t6040_sleep *s) +{ + const u32 appclk = APCIE_PORT_APPCLK_EN | APCIE_PORT_APPCLK_CGDIS; + unsigned int timeouts = 0; + u32 val; + int i; + + for (i = 0; i < ARRAY_SIZE(s->gp_port); i++) { + void __iomem *phy = s->gp_phy[i], *port = s->gp_port[i]; + u32 cfg, app; + + if (!phy || !port || !s->gp_saved[i]) + continue; + cfg = s->gp_phy_saved[i]; + app = s->gp_appclk_saved[i]; + writel(readl(phy) | APCIE_PHY_CTRL_CLK0REQ, phy); + if (readl_poll_timeout_atomic(phy, val, val & APCIE_PHY_CTRL_CLK0ACK, 1, 50000)) + timeouts++; + writel(readl(phy) | APCIE_PHY_CTRL_CLK1REQ, phy); + if (readl_poll_timeout_atomic(phy, val, val & APCIE_PHY_CTRL_CLK1ACK, 1, 50000)) + timeouts++; + if (!(cfg & APCIE_PHY_CTRL_RESET)) { + writel(readl(phy) & ~APCIE_PHY_CTRL_RESET, phy); + udelay(1); + } + writel(readl(phy) | (cfg & (APCIE_PORT_PHY_EN0 | APCIE_PORT_PHY_EN1)), phy); + writel(readl(phy) | (cfg & APCIE_PORT_PHY_LINK), phy); + writel((readl(port + APCIE_PORT_APPCLK) & ~appclk) | (app & appclk), + port + APCIE_PORT_APPCLK); + /* + * The stop also wrote the controller's PERST register 0, and without it the + * port does not report ready. Put it back as it was (the endpoint's PERST# pin + * stays asserted), with the guard bit set around it. + */ + writel(readl(port + APCIE_PORT_PERST_GUARD) | BIT(8), + port + APCIE_PORT_PERST_GUARD); + writel(s->gp_perst_saved[i], port + APCIE_PORT_PERST); + writel(readl(port + APCIE_PORT_PERST_GUARD) & ~BIT(8), + port + APCIE_PORT_PERST_GUARD); + if (readl_poll_timeout_atomic(port + APCIE_PORT_STATUS, val, + val & APCIE_PORT_STATUS_READY, 1, 50000)) + timeouts++; + } + s->gp_clk_timeouts = timeouts; +} + +/* + * The wake return, on the boot core, once the boot object's mailbox has handed it to the + * resume entry. The boot object restores no SoC register: every write here is this driver's. + * In order: the timer and IRQ routing, the stopped CPUs and the AIC, then as the PMGR + * wake restore the complexes back, the two wake pwrgates, SOC-East retention off (with the + * busy poll) and the axi2af fabric-bridge re-arm; what a Linux sleep needs on top, ANS's + * storage link, the GP PCIe ports' clocks and the AOP island's power words; then the CPM + * voters re-asserted, the clk mux restored, the AOP-global S2R control cleared, the + * keep-alive bits back, WD2 armed over the device resume, and the wake-status words read. + */ +static void t6040_sleep_return(struct t6040_sleep *s) +{ + u32 val; + int i, ret; + + /* The reset left the timer and IRQ routing at their EL2 defaults. */ + write_sysreg(s->cntkctl, cntkctl_el1); + isb(); + write_sysreg_s(s->siq_cfg, SYS_APL_SIQ_CFG_EL1); + t6040_wake_sync(); + + spin_table_stop_finished(); + t6040_wake_sync(); + aic_sleep_restore(); + t6040_wake_sync(); + + /* + * The complexes back: E, P0 and P1 to 0xf, the two display complexes to the nibble + * the commit found, with no poll. Register 0xe440f8 of each CPU complex could also + * be written 1. That is not done here: the secondaries come up through the + * normal per-CPU start afterwards, and all 14 CPUs of the J616s are + * online after a wake without it. + */ + for (i = 0; i < CLUSTER_CTL_NR; i++) { + u32 on = i < CLUSTER_CTL_FIRST_WAIT ? CLUSTER_CTL_ON : + s->cluster_saved[i] & CLUSTER_CTL_NIBBLE; + + val = readl(s->cluster_ctl + CLUSTER_CTL(i)); + writel((val & ~CLUSTER_CTL_NIBBLE) | on, s->cluster_ctl + CLUSTER_CTL(i)); + } + t6040_wake_sync(); + + /* Wake pwrgates and CPM voters: read, set BIT31. */ + writel(readl(s->pwrgate + PWRGATE_WAKE_0) | PWRGATE_ON, s->pwrgate + PWRGATE_WAKE_0); + writel(readl(s->pwrgate + PWRGATE_WAKE_1) | PWRGATE_ON, s->pwrgate + PWRGATE_WAKE_1); + t6040_wake_sync(); + + /* SOC-East retention off: clear DEV_DISABLE, then poll BUSY clear. */ + val = readl(s->soc_east + SOC_EAST_RETENTION); + writel(val & ~PS_DEV_DISABLE, s->soc_east + SOC_EAST_RETENTION); + ret = readl_poll_timeout_atomic(s->soc_east + SOC_EAST_RETENTION, val, + !(val & PS_BUSY), 1, SOC_EAST_TIMEOUT_US); + if (ret) + dev_warn(s->dev, "SOC-East retention word still busy on wake (%#x)\n", val); + t6040_wake_sync(); + + /* + * Bit 0 of the SoC's 14 energy accumulators could be set again here. They + * only account energy and nothing in this kernel reads them, so they are + * left as the power loss left them. + */ + + /* The fabric-bridge retention re-arm, or the bridges drop their traffic. */ + t6040_axi2af_rearm(s); + t6040_wake_sync(); + + /* ANS's storage link, after the bridges and before nvme's resume_early (see there). */ + s->apcie_wake_step = 0; + s->pcie_ans_on_ret = 0; + if (s->pcie_ans_stopped) + s->pcie_ans_on_ret = t6040_apcie_st_raise(s); + t6040_gp_pcie_clk(s); + t6040_wake_sync(); + t6040_aop_ps_restore(s); + t6040_wake_sync(); + + writel(readl(s->pwrgate + PWRGATE_CPM_0) | PWRGATE_ON, s->pwrgate + PWRGATE_CPM_0); + writel(readl(s->pwrgate + PWRGATE_CPM_1) | PWRGATE_ON, s->pwrgate + PWRGATE_CPM_1); + writel(readl(s->pwrgate + PWRGATE_CPM_2) | PWRGATE_ON, s->pwrgate + PWRGATE_CPM_2); + t6040_wake_sync(); + + /* Undo the entry clk-mux park; the clock restore is otherwise a NOP. */ + t6040_clkgen_restore(s); + t6040_wake_sync(); + + /* Software ack of the armed S2R request (the S2R timer has no stop). */ + writel(0, s->aop_global + AOP_S2R_CTRL); + t6040_wake_sync(); + + /* The keep-alive bits the commit cleared, back (the wake restore enables them). */ + if (s->ka_on_before) + t6040_keep_alive(s, true, s->ka_on_before); + t6040_wake_sync(); + + /* A longer WD2 over the device resume, until .end. */ + t6040_sleep_set_wd2(s, WDT_SUSPEND_SECS); + t6040_wake_sync(); + + if (s->mbox) { + dcache_inval_poc((unsigned long)s->mbox, (unsigned long)(s->mbox + 1)); + dev_dbg(s->dev, "wake: resumes now %llu\n", READ_ONCE(s->mbox->resumes)); + } + for (i = 0; i < AOP_WAKE_WORDS; i++) + s->wake_status[i] = readl(s->aop_global + AOP_WAKE_STATUS(i)); + s->returned = true; +} + +/* + * Offer "deep" only when the system was booted with allow_entry: without it + * every suspend to RAM is refused, so mem_sleep keeps s2idle as its default. + */ +static int t6040_sleep_valid(suspend_state_t state) +{ + return state == PM_SUSPEND_MEM && READ_ONCE(allow_entry); +} + +/* + * The VDD_CIO rail request (SMC key pmVC) is written in each .begin; its key and value + * belong to the SMC driver (apple_smc_request_pmvc()). + */ + +/* The SMC node is apple,t6041-smc + apple,t8103-smc here (macsmc's match list). */ +static const struct of_device_id t6040_smc_match[] = { + { .compatible = "apple,t8103-smc" }, + { .compatible = "apple,smc" }, + {} +}; + +static void t6040_sleep_pmvc(struct t6040_sleep *s) +{ + struct platform_device *pdev; + struct device_node *np; + struct apple_smc *smc; + int ret = -ENODEV; + + np = of_find_matching_node(NULL, t6040_smc_match); + pdev = np ? of_find_device_by_node(np) : NULL; + of_node_put(np); + smc = pdev ? platform_get_drvdata(pdev) : NULL; + if (smc) + ret = apple_smc_request_pmvc(smc); + if (pdev) + put_device(&pdev->dev); + if (!smc) + dev_warn(s->dev, "pmVC: no SMC to write to: %d\n", ret); +} + +/* + * The refusal-first gate of a suspend to RAM, which t6040_sleep_valid() offers only with + * allow_entry. With a blocker left, print the blocker list and return an error here, before + * any device suspends, so nothing is entered. A dry run proceeds and refuses at .enter. No + * blocker and no dry run: the path that reaches the commit. + */ +static int t6040_sleep_begin(suspend_state_t state) +{ + struct t6040_sleep *s = t6040_sleep; + unsigned int disp = 0; + unsigned int live = t6040_sleep_live_blockers(s, NULL, &disp); + /* + * The display pipes are lowered during device suspend (apple-dcp .suspend), + * so .begin only gates on the non-display live blockers; .enter re-checks the + * full set at the commit point, by which time the display is off. + */ + unsigned int begin_total = live - disp; + unsigned int dry = READ_ONCE(dry_run), secs; + u32 dcs_to; + + /* + * The PM core calls .begin and .end for suspend-to-idle too when there + * are no s2idle_ops. s2idle needs nothing from this controller. + */ + s->mem_begun = false; + if (state != PM_SUSPEND_MEM) + return 0; + s->mem_begun = true; + + t6040_sleep_set_dry(s, 0); + s->returned = false; + s->gp_saved[0] = false; + s->gp_saved[1] = false; + /* + * Decide before anything is written: a refused suspend leaves the AOP islands, the SMC + * rail request and the DCS power words as they were. Only a dry run or a proceeding + * suspend, which both go through device suspend and .enter, needs them set up. + */ + if (begin_total && !dry) { + dev_info(s->dev, "suspend requested: %u live blocker(s) (%u display, cleared during device suspend)\n", + live, disp); + t6040_sleep_log_blockers(s); + dev_info(s->dev, "refusing the suspend; nothing entered\n"); + return -EOPNOTSUPP; + } + + t6040_aop_ps_save(s); + t6040_sleep_pmvc(s); + t6040_dcs_dyn_pg(s); + dcs_to = t6040_dcs_ps_auto_on(s); + if (dcs_to) + dev_warn(s->dev, "DCS power words %#x did not read on before automatic mode\n", + dcs_to); + if (!begin_total && !dry) { + /* + * The non-display blockers are clear and no dry run is asked for: proceed + * to device suspend; .enter holds the real commit until the display is + * also off. The AON WD2 is the net from here. + */ + dev_dbg(s->dev, "entering: %u display pipe(s) on, to be lowered before the commit\n", + disp); + t6040_sleep_power_ops(s); + t6040_sleep_set_wd2(s, WDT_SUSPEND_SECS); + pm_set_suspend_via_firmware(); + return 0; + } + + dev_info(s->dev, "suspend requested: %u live blocker(s) (%u display, cleared during device suspend)\n", + live, disp); + t6040_sleep_log_blockers(s); + + /* A dry run: the refusal case returned above. */ + dry = min(dry, 2U); + t6040_sleep_power_ops(s); + if (!s->power_ops) + dev_warn(s->dev, "dry run: no CPU parking power ops, so no hold at the commit point\n"); + t6040_sleep_set_dry(s, dry); + pm_set_suspend_via_firmware(); + secs = min(READ_ONCE(wd2_secs), WDT_MAX_SECS); + if (secs && s->wdt) + t6040_sleep_set_wd2(s, secs); + else + secs = 0; + dev_info(s->dev, "dry run %u: firmware-driven device sleep short of %s, refused at the commit point, WD2 %u s (0: off)\n", + dry, dry >= 2 ? "the stops only a power cycle undoes" : + "the last steps", secs); + return 0; +} + +/* After the device suspend, in process context. */ +static int t6040_sleep_prepare(void) +{ + t6040_sleep_watch_bound(t6040_sleep); + + return 0; +} + +/* + * A dry run at the commit point, interrupts off and one CPU up: every + * check a real commit makes, the parked CPUs held and let go, at level 2 each + * coprocessor's last check, then the refusal. Never reached: the enclave's + * sleep request, each coprocessor's last stop, the wake masks, the SoC + * power-down. + */ +static int t6040_sleep_dry_enter(struct t6040_sleep *s, unsigned int dry) +{ + u32 p[AOP_WAKE_WORDS]; + unsigned int refused; + int ret, i; + + dev_info(s->dev, "dry run %u at the commit point: %u CPU(s) online\n", dry, + num_online_cpus()); + t6040_sleep_log_s2r_state(s, "dry-run commit point"); + /* + * A wake source pending and left unmasked by the masks a real commit arms would keep the + * AP partition from powering down, and the sleep engine then stalls. Read only. + */ + for (i = 0; i < AOP_WAKE_WORDS; i++) + p[i] = readl(s->aop_global + AOP_WAKE_STATUS(i)) & ~t6040_wake_mask[i]; + dev_info(s->dev, "dry-run commit point: wake status left unmasked by the commit's masks %*ph\n", + (int)sizeof(p), p); + refused = t6040_sleep_refuse(s); + dev_info(s->dev, "a real commit would %s here (%u check(s) failed)\n", + refused ? "refuse" : "go on", refused); + + ret = s->power_ops ? spin_table_hold_parked() : -EOPNOTSUPP; + if (ret) + dev_err(s->dev, "the parked CPUs didn't answer the hold: %d\n", ret); + else + dev_info(s->dev, "%u parked CPU(s) answered the hold\n", + num_present_cpus() - num_online_cpus()); + + if (dry >= 2 && !ret) + dev_info(s->dev, "the coprocessors' last checks: %d\n", + apple_sleep_coprocs_check(s->dev)); + if (!ret) + spin_table_unhold_parked(); + + dev_info(s->dev, "dry run: refused before the enclave's sleep request (%s), the coprocessors' last stops and the SoC power-down\n", + apple_sleep_enclave_registered() ? "left by sep" : "none left"); + + return -EBUSY; +} + +/* + * regmap_read_poll_timeout_atomic() reads ktime_get(), which warns while timekeeping is + * suspended. Counted 1 us steps instead. + */ +static int t6040_regmap_poll_off(struct regmap *map, u32 off, u32 *cur) +{ + unsigned int us; + + for (us = 0; ; us++) { + if (regmap_read(map, off, cur)) + return -EIO; + if (!FIELD_GET(PS_ACTUAL, *cur)) + return 0; + if (us >= PS_OFF_TIMEOUT_US) + return -ETIMEDOUT; + udelay(1); + } +} + +/* A power word behind a regmap (pmgr_disp) or a plain window (soc_east). */ +struct t6040_psw { + struct regmap *map; + void __iomem *io; + u32 off; +}; + +static u32 t6040_psw_read(const struct t6040_psw *w) +{ + u32 v = 0; + + if (w->map) + regmap_read(w->map, w->off, &v); + else + v = readl(w->io + w->off); + return v; +} + +static void t6040_psw_write(const struct t6040_psw *w, u32 v) +{ + if (w->map) + regmap_write(w->map, w->off, v); + else + writel(v, w->io + w->off); +} + +static int t6040_psw_poll_long(const struct t6040_psw *w, u32 mask, u32 want, unsigned int max_us) +{ + unsigned int us; + + for (us = 0; us < max_us; us++) { + if ((t6040_psw_read(w) & mask) == want) + return 0; + udelay(1); + } + return -ETIMEDOUT; +} + +static int t6040_psw_poll(const struct t6040_psw *w, u32 mask, u32 want) +{ + unsigned int us; + + for (us = 0; us < DISP_PS_TIMEOUT_US; us++) { + if ((t6040_psw_read(w) & mask) == want) + return 0; + udelay(1); + } + return -ETIMEDOUT; +} + +/* + * The power-down of a PS word: with + * AUTO_ENABLE set, take it out of automatic mode with the target kept and the flags cleared, and + * wait for ACTUAL 15; then @pre (the SYS hooks); then TARGET 0 with bits 9:8 written 1 and + * bits 10-12, 24-31 kept; then poll bit11 clear and ACTUAL 0. + */ +static int t6040_psw_off(struct t6040_sleep *s, const struct t6040_psw *w, + void (*pre)(struct t6040_sleep *s)) +{ + u32 v = t6040_psw_read(w); + + if (v & PS_AUTO_ENABLE) { + t6040_psw_write(w, v & ~(PS_AUTO_ENABLE | PS_FLAGS)); + t6040_psw_poll(w, PS_ACTUAL, PS_ACTUAL); + v = t6040_psw_read(w); + } + if (pre) + pre(s); + t6040_psw_write(w, (v & ~(PS_FLAGS | PS_TARGET)) | PS_FLAGS); + t6040_psw_poll(w, PS_BUSY, 0); + return t6040_psw_poll(w, PS_ACTUAL, 0); +} + +/* DISP_SYS's hooks before its PS write: the axi2af bridge bit0s, then the retention pair. */ +static void t6040_disp_sys_pre(struct t6040_sleep *s) +{ + void __iomem *bridge = s->axi2af_bridge[AXI2AF_DISP_SYS]; + + if (bridge) { + writel(readl(bridge + AXI2AF_BRIDGE_104) & ~BIT(0), bridge + AXI2AF_BRIDGE_104); + writel(readl(bridge + AXI2AF_BRIDGE_108) & ~BIT(0), bridge + AXI2AF_BRIDGE_108); + } + if (s->disp_pwrgate) { + writel(3, s->disp_pwrgate + DISP_PWRGATE_A); + writel(0, s->disp_pwrgate + DISP_PWRGATE_B); + } +} + +/* dart-dcp's outstanding-transaction counters at 0 before its domain goes. */ +#define DCP_DART_TXN_COUNT GENMASK(20, 16) + +static int t6040_dcp_dart_idle(struct t6040_sleep *s) +{ + static const u16 regs[] = { 0x240, 0x244, 0x250, 0x254, 0x260, 0x270 }; + unsigned int i, us = 0; + + if (!s->dcp_dart) + return 0; + for (i = 0; i < ARRAY_SIZE(regs); i++) + while (readl(s->dcp_dart + regs[i]) & DCP_DART_TXN_COUNT) { + if (++us > DISP_PS_TIMEOUT_US) + return -ETIMEDOUT; + udelay(1); + } + return 0; +} + +/* The internal display's FE and SYS domains, child first, after its sub-blocks. */ +static int t6040_disp_off(struct t6040_sleep *s, const struct t6040_sleep_group *g) +{ + struct t6040_psw cpu = { .map = g->regmap, .off = g->first }; + struct t6040_psw fe = { .io = s->soc_east, .off = PS_DISP_FE }; + struct t6040_psw sys = { .io = s->soc_east, .off = PS_DISP_SYS }; + int ret; + + s->disp_down = true; + /* + * The PMC's bandwidth-request and boost interface enables of the display pipes + * (0x5048ac020 / 0x5048ac028, one bit per pipe; they belong set while a pipe's SYS + * domain is powered). The power loss clears them and nothing sets them + * again: without its boost bit a pipe's backlight data stream never runs. + */ + s->pmc_if_valid = false; + if (s->pmc_bwr) { + s->pmc_if_saved[0] = readl(s->pmc_bwr); + s->pmc_if_saved[1] = readl(s->pmc_bwr + PMC_BOOST); + s->pmc_if_valid = true; + } + /* The PMS DCP voters to 0 first, as T8140's driver ends. */ + if (s->pmc_dcp) { + s->pmc_dcp_saved[0] = readl(s->pmc_dcp); + s->pmc_dcp_saved[1] = readl(s->pmc_dcp + 4); + writel(0, s->pmc_dcp); + writel(0, s->pmc_dcp + 4); + } + /* + * DISP_CPU's word refuses any write from here: rewriting its own value at runtime reset + * the SoC. Its CPU is left in automatic mode and the PMGR gates it once the firmware + * idles in WFI with the run bit kept: wait for ACTUAL 0, after dart-dcp's transactions + * have drained, and go on either way. + */ + if (FIELD_GET(PS_ACTUAL, t6040_psw_read(&cpu))) + t6040_dcp_dart_idle(s); + t6040_psw_poll_long(&cpu, PS_ACTUAL, 0, DISP_CPU_GATE_US); + + ret = t6040_psw_off(s, &fe, NULL); + if (!ret) + ret = t6040_psw_off(s, &sys, t6040_disp_sys_pre); + return ret; +} + +/* + * The raise (the mirror): TARGET 15, poll ACTUAL 15, poll bit11; for DISP_SYS the + * retention pair (A := 0, B := 1) and the axi2af bridge bit0s set; then bits 9:8 cleared and + * AUTO_ENABLE back where it belongs (DISP_CPU, DISP_SYS). + */ +static int t6040_psw_on(struct t6040_sleep *s, const struct t6040_psw *w, bool sys, + bool autoen) +{ + u32 v = t6040_psw_read(w); + int ret; + + t6040_psw_write(w, (v & ~(PS_AUTO_ENABLE | PS_FLAGS)) | PS_TARGET); + ret = t6040_psw_poll(w, PS_ACTUAL, PS_ACTUAL); + t6040_psw_poll(w, PS_BUSY, 0); + if (sys) { + void __iomem *bridge = s->axi2af_bridge[AXI2AF_DISP_SYS]; + + if (s->disp_pwrgate) { + writel(0, s->disp_pwrgate + DISP_PWRGATE_A); + writel(1, s->disp_pwrgate + DISP_PWRGATE_B); + } + if (bridge && !ret) { + writel(readl(bridge + AXI2AF_BRIDGE_104) | BIT(0), + bridge + AXI2AF_BRIDGE_104); + writel(readl(bridge + AXI2AF_BRIDGE_108) | BIT(0), + bridge + AXI2AF_BRIDGE_108); + } + } + v = t6040_psw_read(w); + t6040_psw_write(w, (v & ~PS_AUTO_ENABLE) | PS_FLAGS | (autoen ? PS_AUTO_ENABLE : 0)); + return ret; +} + +/* The internal display back on, parent first, whatever brought .enter back. */ +static void t6040_disp_back(struct t6040_sleep *s) +{ + struct t6040_psw sys = { .io = s->soc_east, .off = PS_DISP_SYS }; + struct t6040_psw fe = { .io = s->soc_east, .off = PS_DISP_FE }; + + if (!s->disp_down) + return; + s->disp_down = false; + /* DISP_CPU's word can't be written from here: automatic mode raises it. */ + if (t6040_psw_on(s, &sys, true, true)) + return; + if (t6040_psw_on(s, &fe, false, false)) + return; + if (s->pmc_dcp) { + writel(s->pmc_dcp_saved[1], s->pmc_dcp + 4); + writel(s->pmc_dcp_saved[0], s->pmc_dcp); + } + if (s->pmc_if_valid) { + u32 bwr = readl(s->pmc_bwr), boost = readl(s->pmc_bwr + PMC_BOOST); + + s->pmc_if_valid = false; + if (bwr != s->pmc_if_saved[0]) + writel(bwr | s->pmc_if_saved[0], s->pmc_bwr); + if (boost != s->pmc_if_saved[1]) + writel(boost | s->pmc_if_saved[1], s->pmc_bwr + PMC_BOOST); + if (bwr != s->pmc_if_saved[0] || boost != s->pmc_if_saved[1]) + dev_dbg(s->dev, "wake: PMC display interfaces BWR %#x boost %#x, put back to %#x %#x\n", + bwr, boost, readl(s->pmc_bwr), readl(s->pmc_bwr + PMC_BOOST)); + } +} + +/* + * The display power words the ADT wants off before a sleep (apple,check-ps-b4-sleep), lowered + * here: after the device suspend, interrupts off, one CPU up, so no driver, interrupt handler + * or DART touches the pipes any more. The DCPs only stop their firmware: lowering the domains + * from their suspend makes the SoC panic. Each group from its last sub-block up to, and not + * including, its pipe CPU, which the PMGR gates by itself: each word as the pmgr-pwrstate + * driver powers a domain off (TARGET 0, the one-shot bits clear), polled to ACTUAL 0. The + * internal display's FE and SYS domains follow. Returns the number of words left on. + */ +static unsigned int t6040_disp_lower(struct t6040_sleep *s) +{ + unsigned int i, left = 0; + + for (i = 0; i < s->ngroups; i++) { + const struct t6040_sleep_group *g = &s->groups[i]; + u32 off = g->last; + + for (;;) { + bool cpu = off == g->first; + u32 val, cur = 0; + + /* An automatically gated CPU still on: wait for the PMGR to gate it. */ + if (cpu && !regmap_read(g->regmap, off, &val) && (val & PS_AUTO_ENABLE) && + FIELD_GET(PS_ACTUAL, val)) + t6040_regmap_poll_off(g->regmap, off, &cur); + if (!cpu && !regmap_read(g->regmap, off, &val) && + t6040_sleep_ps_on(val, cpu)) { + val &= ~(PS_TARGET | PS_AUTO_ENABLE | PS_FLAGS | PS_DEV_DISABLE | + PS_BUSY | PS_UNLATCH); + regmap_write(g->regmap, off, val); + if (t6040_regmap_poll_off(g->regmap, off, &cur)) { + dev_warn(s->dev, "%s +%#x still on: %#x\n", g->name, + off, cur); + left++; + } + } + if (off == g->first) + break; + off -= PS_STRIDE; + } + if (!i && !strcmp(g->name, "disp") && t6040_disp_off(s, g)) + left++; + } + return left; +} + +static int t6040_sleep_enter(suspend_state_t state) +{ + struct t6040_sleep *s = t6040_sleep; + unsigned int dry = s->dry, left; + int ret; + + if (dry) + return t6040_sleep_dry_enter(s, dry); + + left = t6040_disp_lower(s); + if (left) + dev_warn(s->dev, "%u display power word(s) did not go off\n", left); + if (t6040_sleep_live_blockers(s, NULL, NULL)) { + unsigned int on = t6040_sleep_log_groups(s); + + dev_info(s->dev, "at the commit point: %u CPU(s) online, %u display group(s) on; refusing\n", + num_online_cpus(), on); + t6040_disp_back(s); + return -EBUSY; + } + + /* The refusals, the hold and the stops in the generic driver's order. */ + if (t6040_sleep_refuse(s)) { + t6040_disp_back(s); + return -EBUSY; + } + ret = spin_table_hold_parked(); + if (ret) { + t6040_disp_back(s); + return ret; + } + ret = apple_sleep_coprocs_check(s->dev); + if (!ret) + ret = apple_sleep_enclave_stop(s->dev); + if (ret) { + spin_table_unhold_parked(); + t6040_disp_back(s); + return ret; + } + s->pcie_ans_stopped = false; + apple_sleep_coprocs_stop(true, &s->pcie_ans_stopped); + apple_sleep_coprocs_stop(false, &s->pcie_ans_stopped); + t6040_wake_arm(s); + + s->cntkctl = read_sysreg(cntkctl_el1); + pm_set_resume_via_firmware(); + ret = cpu_suspend(0, t6040_sleep_commit); + if (!ret) + t6040_sleep_return(s); + + if (s->returned) + t6040_wake_sync(); + t6040_wake_restore(s); + if (s->returned) { + t6040_wake_sync(); + t6040_sio_ps_restore(s); + } + t6040_disp_back(s); + if (s->returned) { + t6040_wake_sync(); + if (s->pcie_ans_on_ret) + dev_warn(s->dev, "wake: the storage PCIe raise failed after step %u: %d\n", + s->apcie_wake_step, s->pcie_ans_on_ret); + if (s->gp_clk_timeouts) + dev_warn(s->dev, "wake: %u GP PCIe clock or ready poll(s) timed out\n", + s->gp_clk_timeouts); + if (s->aop_ps_timeouts) + dev_warn(s->dev, "wake: %u AOP island power word(s) did not come on\n", + s->aop_ps_timeouts); + } + return ret; +} + +static void t6040_sleep_end(void) +{ + struct t6040_sleep *s = t6040_sleep; + unsigned int dry = s->dry; + + if (!s->mem_begun) + return; + s->mem_begun = false; + t6040_sleep_clear_wd2(s); + if (dry) + dev_info(s->dev, "dry run %u over, WD2 off\n", dry); + t6040_sleep_set_dry(s, 0); +} + +static const struct platform_suspend_ops t6040_sleep_ops = { + .valid = t6040_sleep_valid, + .begin = t6040_sleep_begin, + .prepare = t6040_sleep_prepare, + .enter = t6040_sleep_enter, + .end = t6040_sleep_end, +}; + +static int t6040_sleep_parse_groups(struct t6040_sleep *s) +{ + struct device_node *np = s->dev->of_node; + const char *prop = "apple,check-ps-b4-sleep"; + int n, i; + + /* Each group: the PMGR syscon, the CPU's offset and the last sub-block's. */ + n = of_property_count_u32_elems(np, prop); + if (n <= 0 || n % 3) + return dev_err_probe(s->dev, -EINVAL, "no valid %s\n", prop); + n /= 3; + if (of_property_count_strings(np, "apple,check-ps-b4-sleep-names") != n) + return dev_err_probe(s->dev, -EINVAL, "one name per %s group\n", prop); + + s->groups = devm_kcalloc(s->dev, n, sizeof(*s->groups), GFP_KERNEL); + if (!s->groups) + return -ENOMEM; + + for (i = 0; i < n; i++) { + struct t6040_sleep_group *g = &s->groups[i]; + struct of_phandle_args args; + int ret; + + ret = of_parse_phandle_with_fixed_args(np, prop, 2, i, &args); + if (ret) + return dev_err_probe(s->dev, ret, "%s[%d]\n", prop, i); + g->regmap = syscon_node_to_regmap(args.np); + of_node_put(args.np); + if (IS_ERR(g->regmap)) + return dev_err_probe(s->dev, PTR_ERR(g->regmap), "%s[%d] regmap\n", + prop, i); + g->first = args.args[0]; + g->last = args.args[1]; + if (g->first > g->last || (g->last - g->first) % PS_STRIDE || + (g->last - g->first) / PS_STRIDE >= PS_MAX_PER_GROUP) + return dev_err_probe(s->dev, -EINVAL, "%s[%d]: bad range %#x..%#x\n", + prop, i, g->first, g->last); + of_property_read_string_index(np, "apple,check-ps-b4-sleep-names", i, &g->name); + } + s->ngroups = n; + + return 0; +} + +/* + * A watched device's own power word: its first power domain, a PMGR power-state + * node whose reg is the offset in its parent's syscon. Optional. + */ +static void t6040_sleep_watch_init(struct t6040_sleep *s, unsigned int i, + const char *compatible, const char *name) +{ + struct t6040_sleep_watch *w = &s->watch[i]; + struct device_node *np, *pd, *pmgr; + struct regmap *regmap; + + w->compatible = compatible; + np = of_find_compatible_node(NULL, NULL, compatible); + if (!np) + return; + pd = of_parse_phandle(np, "power-domains", 0); + of_node_put(np); + if (!pd) + return; + pmgr = of_get_parent(pd); + if (pmgr && !of_property_read_u32(pd, "reg", &w->ps_offset)) { + regmap = syscon_node_to_regmap(pmgr); + if (!IS_ERR(regmap)) { + w->ps_regmap = regmap; + w->has_ps = true; + } + } + of_node_put(pmgr); + of_node_put(pd); + if (!w->has_ps) + dev_warn(s->dev, "%s: no power word to watch\n", name); +} + +/* + * A named register block, shared with the owning driver: mapped, not claimed. NULL when it is + * missing from the node or smaller than @min, the extent the driver accesses. + */ +static void __iomem *t6040_sleep_map_min(struct device *dev, const char *name, size_t min) +{ + struct device_node *np = dev->of_node; + struct resource res; + int index; + + index = of_property_match_string(np, "reg-names", name); + if (index < 0 || of_address_to_resource(np, index, &res) || resource_size(&res) < min) + return NULL; + + return t6040_ioremap_np(dev, res.start, resource_size(&res)); +} + +static void __iomem *t6040_sleep_map(struct device *dev, const char *name) +{ + return t6040_sleep_map_min(dev, name, 0); +} + +/* + * ANS's storage-apcie windows + PLL/lane tunables, from the apple,apcie node + * the boot object publishes from the ADT. Our static tree has none + * of these, so until aura fills the node @valid stays false and a real sleep + * with ANS stopped refuses (t6040_sleep_refuse). The reg-names and tunable + * property names match the T8140's apple,apcie shape exactly, so the same + * loader fills both SoCs. The GP PCIe host (pcie@1cb0000000, Wi-Fi and SD) is a + * different node, mapped by t6040_sleep_map_gp(). + */ +static void t6040_sleep_map_apcie(struct t6040_sleep *s) +{ + struct t6040_apcie_st *a = &s->apcie_st; + struct device_node *np = of_parse_phandle(s->dev->of_node, "apple,apcie", 0); + + if (!np) + return; + a->axi = apple_sleep_apcie_block(s->dev, np, "axi2af", "apple,tunable-axi2af", + &a->axi_tunable); + a->cio3pll = apple_sleep_apcie_block(s->dev, np, "cio3pll", "apple,tunable-cio3pll", + &a->cio3pll_tunable); + a->clkgen = apple_sleep_apcie_block(s->dev, np, "clkgen", "apple,tunable-clkgen", + &a->clkgen_tunable); + a->phy = apple_sleep_apcie_block(s->dev, np, "phy-block", "apple,tunable-phy", + &a->phy_tunable); + a->rc = apple_sleep_apcie_block(s->dev, np, "rc", NULL, NULL); + a->phy_ip = apple_sleep_apcie_block(s->dev, np, "phy-ip", "apple,tunable-phy-ip-pll", + &a->phy_ip_pll); + a->phy_ip_lanes = apple_sleep_apcie_tunable(s->dev, np, "phy-ip", + "apple,tunable-phy-ip-auspma"); + of_node_put(np); + + a->valid = a->axi && a->cio3pll && a->clkgen && a->phy && a->rc && a->phy_ip && + a->phy_ip_lanes; + if (!a->valid) + dev_info(s->dev, + "no storage-apcie windows/tunables (apple,apcie): a real sleep with ANS stopped refuses\n"); +} + +/* Maps the mailbox at @pa if it is aura's, or returns NULL. */ +static struct aura_s2r_mailbox *t6040_sleep_mailbox_at(struct t6040_sleep *s, u64 pa) +{ + struct aura_s2r_mailbox *mbox; + + mbox = devm_memremap(s->dev, pa, sizeof(*mbox), MEMREMAP_WB); + if (IS_ERR(mbox)) + return NULL; + if (mbox->magic != AURA_S2R_MAGIC) { + dev_warn(s->dev, "s2r-mailbox magic %#llx is not AURAS2R1\n", mbox->magic); + devm_memunmap(s->dev, mbox); + return NULL; + } + return mbox; +} + +/* The boot object's mailbox, if it published one. Optional (a blocker if absent). */ +static void t6040_sleep_map_mailbox(struct t6040_sleep *s) +{ + struct device_node *chosen = of_find_node_by_path("/chosen"); + u64 pa; + int ret; + + if (!chosen) + return; + ret = of_property_read_u64(chosen, "aurora,s2r-mailbox", &pa); + of_node_put(chosen); + if (ret) + return; + + s->mbox = t6040_sleep_mailbox_at(s, pa); + if (s->mbox) + s->mbox_pa = pa; +} + +#ifdef CONFIG_HIBERNATION +/* + * Restored from a hibernation image, this kernel runs under the aura of the boot that resumed + * it. That aura moved with the boot object, and so did its mailbox: the one mapped at probe was + * the hibernating boot's, and the boot core would come back from a sleep through this boot's. + * Take this boot's address from the device tree aura keeps at its fixed slot. Without it, drop + * the mailbox, so that a suspend refuses for it instead of arming memory that is no longer + * aura's. + */ +static int t6040_sleep_restore(struct device *dev) +{ + struct t6040_sleep *s = dev_get_drvdata(dev); + struct aura_s2r_mailbox *mbox = NULL; + u64 pa = 0; + int ret; + + ret = aura_hibernate_boot_chosen_u64("aurora,s2r-mailbox", &pa); + if (!ret && s->mbox && pa == s->mbox_pa) + return 0; + if (!ret) + mbox = t6040_sleep_mailbox_at(s, pa); + if (s->mbox) + devm_memunmap(dev, s->mbox); + if (mbox) + dev_info(dev, "restored: s2r-mailbox %#llx, was %#llx on the boot that hibernated\n", + pa, s->mbox_pa); + else + dev_warn(dev, "restored: this boot's s2r-mailbox is unknown (%d, %#llx): dropped\n", + ret, pa); + s->mbox = mbox; + s->mbox_pa = mbox ? pa : 0; + return 0; +} +#endif + +/* The GP PCIe host's port and PHY windows and its ports' PERST# pins, for the commit. */ +static void t6040_sleep_map_gp(struct t6040_sleep *s) +{ + static const char *const ports[] = { "port0", "port1" }; + static const char *const phys[] = { "phy0", "phy1" }; + struct device_node *host, *child; + struct device_node *pins = NULL; + struct resource res; + int p, idx; + + s->gp_perst[0] = -1; + s->gp_perst[1] = -1; + host = of_find_compatible_node(NULL, NULL, "apple,t6041-pcie"); + if (!host) + return; + if (!of_device_is_available(host)) + goto out; + for (p = 0; p < ARRAY_SIZE(s->gp_port); p++) { + idx = of_property_match_string(host, "reg-names", ports[p]); + if (idx >= 0 && !of_address_to_resource(host, idx, &res)) + s->gp_port[p] = t6040_ioremap_np(s->dev, res.start, resource_size(&res)); + idx = of_property_match_string(host, "reg-names", phys[p]); + if (idx >= 0 && !of_address_to_resource(host, idx, &res)) + s->gp_phy[p] = t6040_ioremap_np(s->dev, res.start, resource_size(&res)); + } + for_each_available_child_of_node(host, child) { + struct of_phandle_args args; + u32 reg[1]; + + if (of_property_read_u32_array(child, "reg", reg, 1)) + continue; + p = (reg[0] >> 11) & 0x1f; /* the device number of pci@N,0 */ + if (p > 1 || of_parse_phandle_with_args(child, "reset-gpios", "#gpio-cells", 0, + &args)) + continue; + if (args.args_count >= 2 && (!pins || pins == args.np)) { + if (!pins) + pins = of_node_get(args.np); + s->gp_perst[p] = args.args[0]; + s->gp_perst_low[p] = args.args[1] & 1; /* GPIO_ACTIVE_LOW */ + } + of_node_put(args.np); + } + if (pins) { + if (!of_address_to_resource(pins, 0, &res)) + s->gp_pins = t6040_ioremap_np(s->dev, res.start, resource_size(&res)); + of_node_put(pins); + } + dev_dbg(s->dev, "GP PCIe: ports %d/%d phys %d/%d, PERST# pins %d/%d\n", + !!s->gp_port[0], !!s->gp_port[1], !!s->gp_phy[0], !!s->gp_phy[1], + s->gp_perst[0], s->gp_perst[1]); +out: + of_node_put(host); +} + +static int t6040_sleep_probe(struct platform_device *pdev) +{ + struct t6040_sleep *s; + struct dentry *dir; + int ret, i; + + s = devm_kzalloc(&pdev->dev, sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->dev = &pdev->dev; + + ret = t6040_sleep_parse_groups(s); + if (ret) + return ret; + + /* + * The commit's and the wake's register blocks are mandatory: without them a suspend + * could not be entered or undone, and the node always lists them. The axi2af fabric + * bridges are fixed PAs, not DT. + */ + BUILD_BUG_ON(ARRAY_SIZE(t6040_axi2af) != T6040_AXI2AF_DOMAINS); + BUILD_BUG_ON(ARRAY_SIZE(t6040_tvm) != ARRAY_SIZE(((struct t6040_sleep *)0)->tvm_ctl)); + BUILD_BUG_ON(ARRAY_SIZE(t6040_acc_apsc_pa) != + ARRAY_SIZE(((struct t6040_sleep *)0)->acc_apsc)); + s->cpu_start = t6040_sleep_map(&pdev->dev, "cpu-start"); + s->wdt = t6040_sleep_map(&pdev->dev, "wdt"); + s->cluster_ctl = t6040_sleep_map(&pdev->dev, "cluster"); + s->pwrgate = t6040_sleep_map(&pdev->dev, "pwrgate"); + s->clkgen = t6040_sleep_map(&pdev->dev, "clkgen"); + s->spmi0 = t6040_sleep_map(&pdev->dev, "spmi"); + s->spmi_a0 = t6040_sleep_map(&pdev->dev, "spmi-a0"); + s->mastermini = t6040_sleep_map(&pdev->dev, "mastermini"); + s->soc_east = t6040_sleep_map_min(&pdev->dev, "soc-east", SOC_EAST_SIZE); + s->soc_ps = t6040_sleep_map_min(&pdev->dev, "soc-ps", SOC_PS_SIZE); + s->aop_global = t6040_sleep_map(&pdev->dev, "wake"); + if (!s->cpu_start || !s->wdt || !s->cluster_ctl || !s->pwrgate || !s->clkgen || + !s->spmi0 || !s->spmi_a0 || !s->mastermini || !s->soc_east || !s->soc_ps || + !s->aop_global) + return dev_err_probe(&pdev->dev, -ENODEV, + "a register block is missing or not mapped (cpu-start %d wdt %d cluster %d pwrgate %d clkgen %d spmi %d spmi-a0 %d mastermini %d soc-east %d soc-ps %d wake %d)\n", + !!s->cpu_start, !!s->wdt, !!s->cluster_ctl, !!s->pwrgate, + !!s->clkgen, !!s->spmi0, !!s->spmi_a0, !!s->mastermini, + !!s->soc_east, !!s->soc_ps, !!s->aop_global); + { + struct resource wres; + int widx = of_property_match_string(pdev->dev.of_node, "reg-names", "wdt"); + + s->wdt_stub_ok = widx >= 0 && + !of_address_to_resource(pdev->dev.of_node, widx, &wres) && + wres.start == T6040_WDT_PA; + if (!s->wdt_stub_ok) + dev_warn(&pdev->dev, + "the wdt block is not the AON watchdog: no WD2 restart at the wake\n"); + } + s->aop_ps = t6040_ioremap_np(&pdev->dev, T6040_AOP_PS_PA, 0x160); + for (i = 0; i < ARRAY_SIZE(s->dcs); i++) + s->dcs[i] = t6040_ioremap_np(&pdev->dev, 0x220200000ULL + + (i / 4) * 0x2000000ULL + (i % 4) * 0x200000ULL, + 0x2000); + for (i = 0; i < ARRAY_SIZE(s->atc_cfg); i++) + s->atc_cfg[i] = t6040_ioremap_np(&pdev->dev, + 0x383200000ULL + i * 0x8000000ULL, 0x100); + for (i = 0; i < T6040_AXI2AF_DOMAINS; i++) + s->axi2af_bridge[i] = t6040_ioremap_np(&pdev->dev, + t6040_axi2af[i].bridge_pa, 0x200); + t6040_sleep_map_apcie(s); + t6040_sleep_map_mailbox(s); + t6040_sleep_watch_init(s, T6040_WATCH_SEP, "apple,sep", "sep"); + t6040_sleep_watch_init(s, T6040_WATCH_ANS, "apple,t6041-nvme-ans2", "ans"); + /* Fixed PAs: dart-dcp's counters and the internal display's retention pair. */ + s->dcp_dart = t6040_ioremap_np(&pdev->dev, T6040_DCP_DART_PA, 0x1000); + s->disp_pwrgate = t6040_ioremap_np(&pdev->dev, T6040_DISP_PWRGATE_PA, 0x100); + s->pmc_dcp = t6040_ioremap_np(&pdev->dev, T6040_PMC_DCP_PA, 12); + s->pmc_bwr = t6040_ioremap_np(&pdev->dev, T6040_PMC_BWR_PA, 0x10); + s->ka_soc = t6040_ioremap_np(&pdev->dev, T6040_KA_SOC_PA, KA_SIZE); + s->ka_east = t6040_ioremap_np(&pdev->dev, T6040_KA_EAST_PA, KA_SIZE); + s->smc_wdt_deadline = t6040_ioremap_np(&pdev->dev, T6040_SMC_WDT_DEADLINE_PA, 4); + s->agx_gate = t6040_ioremap_np(&pdev->dev, T6040_AGX_GATE_PA, 4); + s->s2r_lo_count = t6040_ioremap_np(&pdev->dev, T6040_S2R_LO_COUNT_PA, 4); + for (i = 0; i < ARRAY_SIZE(t6040_acc_apsc_pa); i++) + s->acc_apsc[i] = t6040_ioremap_np(&pdev->dev, t6040_acc_apsc_pa[i], 8); + for (i = 0; i < ARRAY_SIZE(t6040_tvm); i++) { + s->tvm_ctl[i] = t6040_ioremap_np(&pdev->dev, t6040_tvm[i].ctl, 4); + s->tvm_status[i] = t6040_ioremap_np(&pdev->dev, t6040_tvm[i].status, 4); + } + s->afr_ps = t6040_ioremap_np(&pdev->dev, T6040_AFR_PS_PA, 4); + t6040_sleep_map_gp(s); + t6040_ed_map(&pdev->dev); + + platform_set_drvdata(pdev, s); + t6040_sleep = s; + apple_sleep_registry_ready(); + + dir = debugfs_create_dir("t6040-sleep", NULL); + debugfs_create_file("preflight", 0400, dir, s, &t6040_sleep_preflight_fops); + + /* + * The CPU power ops act only when a commit stops the parked secondaries, and a dry run + * needs them for the hold. Both are suspends to RAM, offered only with allow_entry, so + * only then are they registered (.begin tries again if this failed). Without it kexec + * keeps refusing parking CPUs as before. Parking itself needs no power ops. + */ + if (allow_entry) + t6040_sleep_power_ops(s); + + suspend_set_ops(&t6040_sleep_ops); + dev_info(&pdev->dev, + "refusal-first: %u display groups, storage-apcie %s, s2r-mailbox %s, parking ops %s, dry run %u\n", + s->ngroups, + s->apcie_st.valid ? "mapped" : "absent", + s->mbox ? "present" : "absent", + s->power_ops ? "registered" : "not registered", READ_ONCE(dry_run)); + t6040_sleep_log_s2r_state(s, "probe"); + + return 0; +} + +const struct apple_sleep_part t6040_sleep_part = { + .probe = t6040_sleep_probe, +#ifdef CONFIG_HIBERNATION + .restore = t6040_sleep_restore, +#endif + /* + * Every enclave is taken, in a dry run as in a real sleep. This sleep was tested with + * neither the lid's state at the wake nor USB wake ports. + */ + .registry_flags = APPLE_SLEEP_REG_ENCLAVE_ANY, +}; diff --git a/drivers/soc/apple/sleep-test.c b/drivers/soc/apple/sleep-test.c index d16544746d909..267f11297864b 100644 --- a/drivers/soc/apple/sleep-test.c +++ b/drivers/soc/apple/sleep-test.c @@ -9,6 +9,9 @@ static const char *const apple_sleep_test_tested[] = { "apple,t8140-sleep", "apple,t8132-sleep", +#if APPLE_SLEEP_T6040 + "apple,t6040-sleep", +#endif }; static bool apple_sleep_test_has(const struct apple_sleep_words *all, u32 offset) @@ -77,6 +80,9 @@ static void apple_sleep_test_words(struct kunit *test) }; int i; + /* A SoC with its own sequences has no lists. */ + if (soc->part) + continue; apple_sleep_test_full(test, c, "ps_all", all); apple_sleep_test_full(test, c, "ps_aon_all", aon); for (i = 0; i < ARRAY_SIZE(named); i++) @@ -114,6 +120,9 @@ static void apple_sleep_test_ops(struct kunit *test) const struct apple_sleep_soc *soc = id->data; const struct apple_sleep_ops *ops = soc->ops; + /* A SoC with a part of its own (T6040) brings its sequences there. */ + if (soc->part) + continue; KUNIT_ASSERT_NOT_NULL_MSG(test, ops, "%s has no operations", id->compatible); KUNIT_EXPECT_NOT_NULL(test, ops->deep_wfi_arm); KUNIT_EXPECT_NOT_NULL(test, ops->lower_io); @@ -134,6 +143,8 @@ static void apple_sleep_test_books(struct kunit *test) for (id = apple_sleep_of_match; id->compatible[0]; id++) { const struct apple_sleep_soc *soc = id->data; + if (soc->part) + continue; KUNIT_EXPECT_LE(test, soc->nstray, BITS_PER_LONG); KUNIT_EXPECT_LE(test, soc->ngate_coprocs, ARRAY_SIZE(((struct apple_sleep *)0)->gate)); @@ -148,6 +159,8 @@ static void apple_sleep_test_wake_names(struct kunit *test) const struct apple_sleep_soc *soc = id->data; int i, j; + if (soc->part) + continue; KUNIT_EXPECT_NE(test, soc->wake_last_word, 0); for (i = 0; i < soc->nwake_names; i++) { u32 n = soc->wake_names[i].id; @@ -171,6 +184,8 @@ static void apple_sleep_test_layout(struct kunit *test) for (id = apple_sleep_of_match; id->compatible[0]; id++) { const struct apple_sleep_soc *soc = id->data; + if (soc->part) + continue; KUNIT_EXPECT_GE(test, soc->ncluster_cpu, 1); KUNIT_EXPECT_LE(test, soc->ncluster_cpu, SLEEP_CPU_CLUSTERS_MAX); KUNIT_EXPECT_GT(test, soc->cpu_start_stride, 0); diff --git a/drivers/soc/apple/sleep.c b/drivers/soc/apple/sleep.c index 202c07c098947..15b844d046b42 100644 --- a/drivers/soc/apple/sleep.c +++ b/drivers/soc/apple/sleep.c @@ -30,6 +30,10 @@ * T8132 takes the same path with its own tables and operations, t8132_sleep_ops, * measured the same way from a macOS sleep under aura's hypervisor. Its blocks are * T8140's at the same offsets from a SoC base 0x80000000 higher unless noted. + * + * A SoC whose sequences differ in more than its lists brings them as a part + * (sleep-internal.h), and shares the registry of coprocessors with the rest: + * today T6040, in sleep-t6040.c. */ #include @@ -59,6 +63,7 @@ #include #include #include +#include #include #include #include @@ -78,34 +83,11 @@ #include #include -/* Must match aura's struct aura_s2r_mailbox. */ -#define AURA_S2R_MAGIC 0x3152325341525541ULL /* "AURAS2R1" */ - -struct aura_s2r_mailbox { - u64 magic; /* aura, before the kernel starts */ - u64 boot_mpidr; /* aura: the only core that may take resume_pa */ - u64 resume_pa; /* kernel, armed before a commit; aura clears it */ - u64 resumes; /* aura: resume_pa taken this many times */ -}; - -/* PS word */ -#define PS_AUTO_ENABLE BIT(28) -#define PS_UNLATCH BIT(12) -#define PS_DEV_DISABLE BIT(10) -#define PS_FLAGS GENMASK(9, 8) -#define PS_ACTUAL GENMASK(7, 4) -#define PS_TARGET GENMASK(3, 0) +#include "sleep-internal.h" /* Clock protection, per slice: its control word, and the bit that guards the slice's range. */ #define PS_PROT_SLICE(n) (0x100 + 0x40 * (n)) #define PS_PROT_SLICE_ON BIT(4) -#define PS_OFF_TIMEOUT_US 5000 -#define PS_BUSY_SETTLE_US 1000 - -/* CPU start block, as aura's smp_start_cpu() uses it. A core's bit is the SoC's. */ -#define CPU_START_STOP 0x0 -#define CPU_START_SYS 0x4 -#define CPU_START_CLUSTER(n) (0x8 + (n) * 4) /* Power-gate page: reads back this value already. Purpose unknown. */ #define PWRGATE_ON 0x80000000 @@ -121,41 +103,12 @@ struct aura_s2r_mailbox { #define CLUSTER_TIMEOUT_US 5000 -/* Watchdog block, ADT /arm-io/wdt reg[0]. WD2 is the sleep safety net. */ -#define WDT_WD0_COUNT 0x00 -#define WDT_WD0_CTL 0x0c -#define WDT_WD1_COUNT 0x10 -#define WDT_WD2_COUNT 0x20 -#define WDT_WD2_ALARM 0x28 -#define WDT_WD2_CTL 0x2c -#define WDT_TICKS_PER_S 24000000 -/* The sleep's own safety net: ADT /arm-io/wdt simple-reconfig-wdog-icc-time. */ -#define WDT_SLEEP_SECS 5 -/* Device suspend and resume can take a while, and nothing else recovers a hang in them. */ -#define WDT_SUSPEND_SECS 120 - #define AON_AP_DEADLINE 0x08 /* Wake sources: a mask, so 0 enables. Status words are sticky. Where and how many: the SoC. */ #define WAKE_WORDS 9 #define WAKE_SOURCES_MAX 32 -/* The AP queue of the PMU's SPMI controller, and its response vector's enable and ack */ -#define SPMI_STATUS 0x00 -#define SPMI_STATUS_RX_EMPTY BIT(24) -#define SPMI_CMD 0x04 -#define SPMI_REPLY 0x08 -#define SPMI_NOTIFY_ENABLE 0x40 -#define SPMI_NOTIFY_REPLY BIT(0) -#define SPMI_NOTIFY_ACK 0x80 -#define SPMI_CMD_ACTIVE BIT(15) -#define SPMI_OP_EXT_WRITE(n) ((n) - 1) -#define SPMI_OP_EXT_READ(n) (0x20 | ((n) - 1)) -#define SPMI_OP_ZERO_WRITE 0x80 -#define SPMI_OP_SLEEP 0x11 -#define SPMI_OP_WAKEUP 0x13 -#define SPMI_POLL_US 20000 - /* A port controller on SPMI: registers go through a page select, then a data window. */ #define HPM_REG_SELECT 0x00 #define HPM_REG_DATA 0x20 @@ -176,9 +129,6 @@ struct aura_s2r_mailbox { #define HPM_CONTRACT_MS 1500 #define HPM_RESET_TRIES 3 -/* A one-byte register write on the sleep's SPMI bus, to the PMU's USID */ -#define SPMI_CMD_WRITE (BIT(15) | 0x30) -#define SPMI_CMD_USID GENMASK(11, 8) /* T8132's power words where they aren't T8140's */ #define T8132_PS_ATC0_COMMON 0x378 @@ -232,99 +182,8 @@ struct aura_s2r_mailbox { #define AMCC_WORD0_DSID_AGE_OUT BIT(25) #define AMCC_WORD1_UNEXP_RT_HIT BIT(8) -/* A coprocessor's ASC */ -#define ASC_CPU_CONTROL 0x44 -#define ASC_CPU_CONTROL_RUN BIT(4) -#define ASC_CPU_STATUS 0x48 - -/* The part of the PCIe block that ANS shares, whether or not its host is bound. */ -#define APCIE_AXI_EN0 0x104 -#define APCIE_AXI_EN1 0x108 -#define APCIE_CLK_CTRL 0x0 -#define APCIE_CLK_BUSY BIT(31) -#define APCIE_CLK_RUN BIT(5) -#define APCIE_CLK_OFF BIT(1) -#define APCIE_CLK_OFF2 BIT(2) -#define APCIE_CLK_START 0x4 -#define APCIE_CLK_START_GO BIT(31) -#define APCIE_PHY_COMMON 0x4000 -#define APCIE_PHY_CTRL 0x8000 -#define APCIE_PHY_CTRL_CLK0REQ BIT(0) -#define APCIE_PHY_CTRL_CLK1REQ BIT(1) -#define APCIE_PHY_CTRL_CLK0ACK BIT(2) -#define APCIE_PHY_CTRL_CLK1ACK BIT(3) -#define APCIE_PHY_CTRL_RESET BIT(4) -#define APCIE_PHY_CTRL_UP BIT(27) -#define APCIE_PHY_RUN 0x8004 -#define APCIE_PHY_RUN_EN BIT(0) -#define APCIE_PHY_RUN_EN2 BIT(4) -#define APCIE_PHY_READY 0x8008 -#define APCIE_PHY_IP_PLL_CTL 0x90 -#define APCIE_PHY_IP_PLL_HOLD BIT(16) -#define APCIE_PORT_INTSTAT 0x100 -#define APCIE_PORT_INTMSK 0x104 -#define APCIE_PORT_INT_LINK GENMASK(7, 4) -#define APCIE_PORT_L2 0x88 -#define APCIE_PORT_L2_ENTER 0x11 -#define APCIE_PORT_L2_BUSY BIT(0) -#define APCIE_PORT_PERST_GUARD 0x13c -#define APCIE_PORT_LINKSTS 0x208 -#define APCIE_PORT_LINKSTS_UP BIT(0) -#define APCIE_PORT_APPCLK 0x800 -#define APCIE_PORT_APPCLK_EN BIT(0) -#define APCIE_PORT_APPCLK_CGDIS BIT(8) -#define APCIE_PORT_PERST 0x82c -#define APCIE_PORT_PHY_LINK BIT(31) -#define APCIE_PORT_PHY_EN0 BIT(9) -#define APCIE_PORT_PHY_EN1 BIT(10) -#define APCIE_ENGINE_CTRL 0x0 -#define APCIE_ENGINE_IRQ_MASK 0x8 -#define APCIE_ENGINE_STOP BIT(0) -#define APCIE_FABRIC_STOP 0x4 -#define APCIE_PORT_STOP_US 5000 -#define APCIE_RC_ZERO 0x04 -#define APCIE_RC_READY_START 0x50 -#define APCIE_RC_READY_CFG 0x54 -#define APCIE_RC_READY_STAT 0x58 -#define APCIE_POLL_US 50000 - -/* The coprocessors the last mile stops. Each driver idles its own first. */ -#define SLEEP_COPROCS 4 - -static struct apple_sleep_coproc { - struct device *dev; - void __iomem *asc; - bool pcie_ans; - int (*last_check)(void *data); - int (*last_stop)(void *data); - void *data; - bool ready; - bool stopped; - u32 status; - int stop_ret; - /* No ASC: its last stop takes it whole, power word @ps included, before the others. */ - bool first; - u32 ps; -} apple_sleep_coprocs[SLEEP_COPROCS]; -static DEFINE_MUTEX(apple_sleep_coprocs_lock); - -/* Irreversible enclave stop; nonzero is legal only if SLEEP was not sent. */ -static struct { - struct device *dev; - int (*stop)(void *data); - void *data; -} apple_sleep_enclave; - -/* - * Written with bits 0 and 63 by every core right before its last WFI, measured - * from macOS. This is not CYC_OVRD, which is undefined on this SoC. - */ -#define SYS_APL_DEEP_SLEEP sys_reg(3, 5, 15, 6, 2) -#define DEEP_SLEEP_ENABLE (BIT_ULL(0) | BIT_ULL(63)) - -#define SYS_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) -#define IPI_SR_PENDING BIT(0) -#define SYS_APL_SIQ_CFG_EL1 sys_reg(3, 4, 15, 10, 4) +/* The coprocessor and enclave registry is shared with the other SoCs' controllers. */ +#define SLEEP_COPROCS APPLE_SLEEP_COPROCS #define SLEEP_DISPLAYS_MAX 3 @@ -527,6 +386,8 @@ struct apple_sleep_soc { /* The PMC whose cluster vote for the encoders needs the complex up, and its bit */ const char *ave_pmc; u32 ave_pmc_cluster; + /* The SoC's own sequences, in place of the lists above, or NULL */ + const struct apple_sleep_part *part; }; struct apple_sleep { @@ -689,17 +550,6 @@ struct apple_sleep { static struct apple_sleep *apple_sleep; -static struct apple_sleep_coproc *apple_sleep_coproc_find(struct device *dev) -{ - int i; - - for (i = 0; i < SLEEP_COPROCS; i++) - if (apple_sleep_coprocs[i].dev == dev) - return &apple_sleep_coprocs[i]; - - return NULL; -} - /* A registered coprocessor its driver has readied, whose run bit the last step clears. */ static bool apple_sleep_coproc_ready_node(const struct device_node *np) { @@ -720,20 +570,6 @@ void apple_sleep_usb_set_asleep(bool asleep) } EXPORT_SYMBOL_GPL(apple_sleep_usb_set_asleep); -static unsigned long sleep_usb_wake_ports; - -void apple_sleep_usb_set_wake_ports(unsigned long ports) -{ - WRITE_ONCE(sleep_usb_wake_ports, ports); -} -EXPORT_SYMBOL_GPL(apple_sleep_usb_set_wake_ports); - -bool apple_sleep_usb_port_can_wake(unsigned int port) -{ - return port < BITS_PER_LONG && (READ_ONCE(sleep_usb_wake_ports) & BIT(port)); -} -EXPORT_SYMBOL_GPL(apple_sleep_usb_port_can_wake); - /* The USB host's vote, kept until the word is mapped. Only bit 0 is touched, as on macOS. */ static DEFINE_SPINLOCK(sleep_slp_ddr_lock); static bool sleep_slp_ddr_usb; @@ -897,200 +733,23 @@ void apple_sleep_restore_pins(void) } EXPORT_SYMBOL_GPL(apple_sleep_restore_pins); -/* Below EL2 a monitor owns the last mile, and this code doesn't run it. */ -int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie_ans, - int (*last_stop)(void *data), void *data) -{ - struct apple_sleep_coproc *c; - - if (!is_kernel_in_hyp_mode()) - return -ENODEV; - - guard(mutex)(&apple_sleep_coprocs_lock); - if (apple_sleep_coproc_find(dev)) - return -EEXIST; - c = apple_sleep_coproc_find(NULL); - if (!c) - return -ENOSPC; - c->asc = asc; - c->pcie_ans = pcie_ans; - c->last_check = NULL; - c->last_stop = last_stop; - c->data = data; - c->ready = false; - c->stopped = false; - c->first = false; - c->ps = 0; - c->dev = dev; - - return 0; -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_register); - -int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), - void *data) -{ - struct apple_sleep_coproc *c; - struct device_node *np; - u32 ps = 0; - int ret; - - if (!last_stop) - return -EINVAL; - np = of_parse_phandle(dev->of_node, "power-domains", 0); - if (np) { - of_property_read_u32(np, "reg", &ps); - of_node_put(np); - } - ret = apple_sleep_coproc_register(dev, NULL, false, last_stop, data); - if (ret) - return ret; - - guard(mutex)(&apple_sleep_coprocs_lock); - c = apple_sleep_coproc_find(dev); - if (c) { - c->first = true; - c->ps = ps; - } - - return 0; -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_register_first); - -int apple_sleep_coproc_set_last_check(struct device *dev, int (*last_check)(void *data)) -{ - struct apple_sleep_coproc *c; - - guard(mutex)(&apple_sleep_coprocs_lock); - c = apple_sleep_coproc_find(dev); - if (!c) - return -ENOENT; - c->last_check = last_check; - - return 0; -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_last_check); - -int apple_sleep_enclave_register(struct device *dev, int (*stop)(void *data), void *data) -{ - if (!is_kernel_in_hyp_mode()) - return -ENODEV; - - if (!stop) - return -EINVAL; - if (!apple_sleep_commit_stops_coprocs()) - return -EOPNOTSUPP; - - guard(mutex)(&apple_sleep_coprocs_lock); - if (apple_sleep_enclave.dev) - return -EBUSY; - apple_sleep_enclave.stop = stop; - apple_sleep_enclave.data = data; - apple_sleep_enclave.dev = dev; - - return 0; -} -EXPORT_SYMBOL_GPL(apple_sleep_enclave_register); - -void apple_sleep_enclave_unregister(struct device *dev) -{ - guard(mutex)(&apple_sleep_coprocs_lock); - if (apple_sleep_enclave.dev == dev) - memset(&apple_sleep_enclave, 0, sizeof(apple_sleep_enclave)); -} -EXPORT_SYMBOL_GPL(apple_sleep_enclave_unregister); - -void apple_sleep_coproc_unregister(struct device *dev) -{ - struct apple_sleep_coproc *c; - - guard(mutex)(&apple_sleep_coprocs_lock); - c = apple_sleep_coproc_find(dev); - if (c) - memset(c, 0, sizeof(*c)); -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_unregister); - -void apple_sleep_coproc_set_ready(struct device *dev, bool ready) -{ - struct apple_sleep_coproc *c; - - guard(mutex)(&apple_sleep_coprocs_lock); - c = apple_sleep_coproc_find(dev); - if (!c) - return; - c->ready = ready; - c->stopped = false; -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_set_ready); - -bool apple_sleep_coproc_stopped(struct device *dev) -{ - struct apple_sleep_coproc *c; - - guard(mutex)(&apple_sleep_coprocs_lock); - c = apple_sleep_coproc_find(dev); - return c && c->stopped; -} -EXPORT_SYMBOL_GPL(apple_sleep_coproc_stopped); - -bool apple_sleep_commit_stops_coprocs(void) -{ - return READ_ONCE(apple_sleep) && pm_suspend_via_firmware(); -} -EXPORT_SYMBOL_GPL(apple_sleep_commit_stops_coprocs); - /* - * The last thing that can fail on the way to the commit: no CPU but this one is up and - * interrupts are off, so no lock can be held. A coprocessor stopped here comes back only - * through a real sleep, so every refusal has to be behind this, and nothing after it may - * go back. In the order measured on macOS, on ANS itself: whatever its driver does last, - * status and control read, then 0 written twice. + * The dry run level of the suspend in progress, for a SoC's part that has dry runs: every + * sleep step that can be taken back runs, and the sleep is refused before the commit. 0 + * otherwise. */ -static void apple_sleep_coproc_run_off(struct apple_sleep_coproc *c) -{ - /* Its own last stop cleared its run bits. */ - if (!c->asc) { - c->stopped = true; - return; - } - c->status = readl(c->asc + ASC_CPU_STATUS); - writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); - writel(readl(c->asc + ASC_CPU_CONTROL) & ~ASC_CPU_CONTROL_RUN, c->asc + ASC_CPU_CONTROL); - c->stopped = true; -} +static unsigned int apple_sleep_dry; -/* A power word that a ready coprocessor's last stop lowers: on until the commit. */ -static bool apple_sleep_coproc_lowers(u32 ps) +void apple_sleep_set_dry(unsigned int level) { - int i; - - for (i = 0; i < SLEEP_COPROCS; i++) - if (apple_sleep_coprocs[i].dev && apple_sleep_coprocs[i].ready && - apple_sleep_coprocs[i].first && apple_sleep_coprocs[i].ps == ps) - return true; - return false; + WRITE_ONCE(apple_sleep_dry, level); } -/* The last refusals, each coprocessor's own: nothing has stopped yet. */ -static int apple_sleep_coprocs_check(struct apple_sleep *s) +bool apple_sleep_power_loss(void) { - int i, ret; - - for (i = 0; i < SLEEP_COPROCS; i++) { - struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; - - if (!c->dev || !c->ready || c->stopped || !c->last_check) - continue; - ret = c->last_check(c->data); - if (ret) { - dev_err(s->dev, "refused: %s: %d\n", dev_name(c->dev), ret); - return ret; - } - } - - return 0; + return pm_suspend_via_firmware() && !READ_ONCE(apple_sleep_dry); } +EXPORT_SYMBOL_GPL(apple_sleep_power_loss); /* * The other cores stop at the commit, and one left awake there keeps the engine waiting @@ -1107,46 +766,6 @@ static int apple_sleep_cpus_hold(struct apple_sleep *s) return ret; } -/* A nonzero result must mean no SLEEP was sent; post-send errors are fatal. */ -static int apple_sleep_enclave_stop(struct apple_sleep *s) -{ - int ret; - - if (!apple_sleep_enclave.stop) - return 0; - ret = apple_sleep_enclave.stop(apple_sleep_enclave.data); - if (ret) - dev_err(s->dev, "refused before SEP SLEEP was sent: %d\n", ret); - - return ret; -} - -/* - * In two passes, after the last refusal: from the first stop only a real sleep restarts the - * GPU, so a failure here goes on to the commit. Those registered first stop whole first, - * before macOS's off words drop under them: the GPU withdraws its PMP request there, and the - * PMP has to see that before its own stop. The rest after the match. - */ -static void apple_sleep_coprocs_stop(struct apple_sleep *s, bool first) -{ - int i; - - if (first) - s->pcie_ans_stopped = false; - for (i = 0; i < SLEEP_COPROCS; i++) { - struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; - - if (!c->dev || !c->ready || c->first != first) - continue; - s->pcie_ans_stopped |= c->pcie_ans; - if (c->stopped) - continue; - c->stop_ret = c->last_stop ? c->last_stop(c->data) : 0; - apple_sleep_coproc_run_off(c); - } - dsb(sy); -} - /* MPIDR's cluster is affinity 2, its core affinity 0, as 0.0.n and 1.1.n on T8140. */ static unsigned int sleep_cpu_cluster(u64 mpidr) { @@ -1246,27 +865,14 @@ static void sleep_ps_lower(struct apple_sleep *s, u32 offset) sleep_ps_lower_finish(s, offset, sleep_ps_lower_start(s, offset)); } -/* - * As macOS raises a domain: the target, then AUTO_ENABLE once it reads on. Measured - * byte for byte on ANS and PCIe, for instance 0x0f002300, 0x0f00200f, 0x1f00200f. - */ static int sleep_ps_raise_start(struct apple_sleep *s, u32 offset) { - void __iomem *reg = s->ps + offset; - u32 now; - - writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET, reg); - - return readl_poll_timeout_atomic(reg, now, (now & PS_ACTUAL) == PS_ACTUAL, 1, - PS_OFF_TIMEOUT_US); + return apple_sleep_ps_raise_start(s->ps + offset); } static void sleep_ps_raise_finish(struct apple_sleep *s, u32 offset) { - void __iomem *reg = s->ps + offset; - - writel((readl(reg) & ~(PS_AUTO_ENABLE | GENMASK(11, 0))) | PS_TARGET | PS_AUTO_ENABLE, - reg); + apple_sleep_ps_raise_finish(s->ps + offset); } static int sleep_ps_raise(struct apple_sleep *s, u32 offset) @@ -1385,7 +991,6 @@ EXPORT_SYMBOL_GPL(apple_sleep_domain_is_down); * target 4 and 0, released with the domain off, then raised again, its DARTs' state kept. The * caller has cleared the run bit, and starts the firmware after. */ -#define PS_BUSY BIT(11) #define PS_RESET BIT(31) #define PS_ENGINE_RESET_4 0x8f000244 #define PS_ENGINE_RESET_0 0x80000000 @@ -1739,20 +1344,8 @@ static int apple_sleep_refuse(struct apple_sleep *s) bool refused = false, idled = false; int i; - for (i = 0; i < SLEEP_COPROCS; i++) { - struct apple_sleep_coproc *c = &apple_sleep_coprocs[i]; - - if (!c->dev) - continue; - if (c->ready) { - idled |= c->pcie_ans; - } else { - /* Nothing may lower a coprocessor's domain under its driver. */ - dev_err(s->dev, "refused: %s is running and its driver has not quiesced it\n", - dev_name(c->dev)); - refused = true; - } - } + if (apple_sleep_coprocs_refuse(s->dev, &idled)) + refused = true; if (num_online_cpus() != 1 || (read_cpuid_mpidr() & MPIDR_HWID_BITMASK) != s->mbox->boot_mpidr) { @@ -2097,14 +1690,6 @@ bool apple_sleep_debug_link_cut(void) } EXPORT_SYMBOL_GPL(apple_sleep_debug_link_cut); -bool apple_sleep_debug_link_up(void) -{ - struct apple_sleep *s = apple_sleep; - - return s && !s->usb_down_ran && sleep_debug_link_up(s); -} -EXPORT_SYMBOL_GPL(apple_sleep_debug_link_up); - /* * macOS takes the wake for a lid closing during the sleep as a dark one: every driver wakes, * the display stays off, and the root domain sleeps again as soon as it sees the lid closed. @@ -2120,36 +1705,6 @@ void apple_sleep_lid_closed_wake(void) } EXPORT_SYMBOL_GPL(apple_sleep_lid_closed_wake); -/* Read in the SMC's noirq resume. Forgotten as each sleep begins. */ -static int sleep_lid_closed = -ENODATA; - -void apple_sleep_lid_at_wake(bool closed) -{ - WRITE_ONCE(sleep_lid_closed, closed); -} -EXPORT_SYMBOL_GPL(apple_sleep_lid_at_wake); - -int apple_sleep_lid_closed_at_wake(void) -{ - return READ_ONCE(sleep_lid_closed); -} -EXPORT_SYMBOL_GPL(apple_sleep_lid_closed_at_wake); - -/* Set in the SMC's noirq resume. Forgotten as each sleep begins. */ -static int sleep_radio_woke = -ENODATA; - -void apple_sleep_radio_at_wake(bool woke) -{ - WRITE_ONCE(sleep_radio_woke, woke); -} -EXPORT_SYMBOL_GPL(apple_sleep_radio_at_wake); - -int apple_sleep_radio_woke(void) -{ - return READ_ONCE(sleep_radio_woke); -} -EXPORT_SYMBOL_GPL(apple_sleep_radio_woke); - /* * The SMC woke the AP for the port alone, and this sleep cut the debug link: the cut's own * events. Sleep again. The link stays down until tipd rearms it at a full wake, so the next @@ -2297,21 +1852,6 @@ static void apple_sleep_lower_apcie(struct apple_sleep *s) sleep_ps_lower(s, d->ans); } -/* Read-modify-write in table order, with no write where nothing changes. */ -static void sleep_tunable_apply_changed(void __iomem *regs, const struct apple_tunable *t) -{ - size_t i; - - for (i = 0; i < t->sz; i++) { - void __iomem *reg = regs + t->values[i].offset; - u32 old = readl(reg); - u32 val = (old & ~t->values[i].mask) | (t->values[i].value & t->values[i].mask); - - if (val != old) - writel(val, reg); - } -} - /* * The power loss takes the PHY's PLL and lane settings with it, and ANS's link can't * train without them. Applied as macOS does: the PLL table, then the lanes, then the @@ -2321,8 +1861,8 @@ static void sleep_apcie_phy_ip_apply(struct apple_sleep *s) { void __iomem *ctl = s->apcie_phy_ip + APCIE_PHY_IP_PLL_CTL; - sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_pll); - sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_lanes); + apple_sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_pll); + apple_sleep_tunable_apply_changed(s->apcie_phy_ip, s->apcie_phy_ip_lanes); writel(readl(ctl) & ~APCIE_PHY_IP_PLL_HOLD, ctl); } @@ -2422,13 +1962,9 @@ static void apple_sleep_lower_io(struct apple_sleep *s) static void apple_sleep_set_wd2(struct apple_sleep *s, unsigned int secs) { - writel(0, s->wdt + WDT_WD2_CTL); - writel(0, s->wdt + WDT_WD2_COUNT); - writel(secs * WDT_TICKS_PER_S, s->wdt + WDT_WD2_ALARM); - writel(1, s->wdt + WDT_WD2_CTL); + apple_sleep_wd2_arm(s->wdt, secs); } -#define PS_PARENT_MISSING BIT(11) #define PS_AUTO_STATE GENMASK(27, 24) #define PS_REWAKE_PASSES 2 @@ -2460,7 +1996,7 @@ static int sleep_ps_find(const u32 *all, int n, u32 offset) */ static int sleep_ps_restore(void __iomem *reg, u32 saved) { - u32 keep = (saved & ~(PS_AUTO_ENABLE | PS_AUTO_STATE | PS_PARENT_MISSING | PS_DEV_DISABLE | + u32 keep = (saved & ~(PS_AUTO_ENABLE | PS_AUTO_STATE | PS_BUSY | PS_DEV_DISABLE | PS_FLAGS | PS_ACTUAL)) | (readl(reg) & PS_AUTO_STATE); u32 now; int ret; @@ -2601,21 +2137,9 @@ static void sleep_ps_rewake_report(struct apple_sleep *s) #define T8132_PS_AON_ON_STATE 4 -/* As macOS writes the PMU: the reply's vector masked, then acknowledged, so none is left raised. */ static void sleep_pmu_write(struct apple_sleep *s, u16 reg, u8 val) { - u32 status, enable; - - enable = readl(s->spmi + SPMI_NOTIFY_ENABLE); - writel(enable & ~SPMI_NOTIFY_REPLY, s->spmi + SPMI_NOTIFY_ENABLE); - writel(((u32)reg << 16) | SPMI_CMD_WRITE | - FIELD_PREP(SPMI_CMD_USID, s->soc->pmu_usid), s->spmi + SPMI_CMD); - writel(val, s->spmi + SPMI_CMD); - if (!readl_poll_timeout_atomic(s->spmi + SPMI_STATUS, status, - !(status & SPMI_STATUS_RX_EMPTY), 1, 1000)) - readl(s->spmi + SPMI_REPLY); - writel(1, s->spmi + SPMI_NOTIFY_ACK); - writel(enable, s->spmi + SPMI_NOTIFY_ENABLE); + apple_sleep_pmu_write(s->spmi, s->soc->pmu_usid, reg, val); } /* @@ -3345,19 +2869,19 @@ static int apple_sleep_enter(suspend_state_t state) if (!ret) ret = sleep_display_off(s); if (!ret) - ret = apple_sleep_coprocs_check(s); + ret = apple_sleep_coprocs_check(s->dev); /* Cover the platform's own preparation, after the PM core's early check. */ if (!ret && pm_wakeup_pending()) ret = -EBUSY; if (!ret) - ret = apple_sleep_enclave_stop(s); + ret = apple_sleep_enclave_stop(s->dev); if (ret) { dev_dbg(s->dev, "refused before any commit coprocessor stop: %d\n", ret); spin_table_unhold_parked(); apple_sleep_stray_restore(s); return ret; } - apple_sleep_coprocs_stop(s, true); + apple_sleep_coprocs_stop(true, &s->pcie_ans_stopped); sleep_wake_arm(s); apple_sleep_match(s); @@ -3365,7 +2889,7 @@ static int apple_sleep_enter(suspend_state_t state) /* Acknowledge a leftover done flag, as the DCS word has none on macOS. */ if (s->soc->ack_cluster_dcs && (readl(s->cluster_dcs) & CLUSTER_UP_DONE)) writel(readl(s->cluster_dcs), s->cluster_dcs); - apple_sleep_coprocs_stop(s, false); + apple_sleep_coprocs_stop(false, &s->pcie_ans_stopped); /* Done by now unless no SMC driver is bound. */ apple_sleep_usb_down(s); s->deep_once = READ_ONCE(sleep_deep_requested); @@ -3393,8 +2917,7 @@ static int apple_sleep_begin(suspend_state_t state) { /* The last sleep's cut is its port controller's to take, by its .complete. */ WRITE_ONCE(sleep_debug_link_cut, false); - WRITE_ONCE(sleep_lid_closed, -ENODATA); - WRITE_ONCE(sleep_radio_woke, -ENODATA); + apple_sleep_wake_info_forget(); /* The PM core also calls .begin for s2idle when no s2idle ops are installed. */ if (state != PM_SUSPEND_MEM) return 0; @@ -3904,9 +3427,8 @@ static int apple_sleep_map_mailbox(struct apple_sleep *s) return 0; } -static struct apple_tunable *apple_sleep_apcie_tunable(struct apple_sleep *s, - struct device_node *np, - const char *name, const char *prop) +struct apple_tunable *apple_sleep_apcie_tunable(struct device *dev, struct device_node *np, + const char *name, const char *prop) { int idx = of_property_match_string(np, "reg-names", name); struct apple_tunable *t; @@ -3914,21 +3436,21 @@ static struct apple_tunable *apple_sleep_apcie_tunable(struct apple_sleep *s, if (idx < 0 || of_address_to_resource(np, idx, &res)) return NULL; - t = devm_apple_tunable_parse(s->dev, np, prop, &res); + t = devm_apple_tunable_parse(dev, np, prop, &res); return IS_ERR(t) ? NULL : t; } -static void __iomem *apple_sleep_apcie_block(struct apple_sleep *s, struct device_node *np, - const char *name, const char *prop, - struct apple_tunable **tunable) +void __iomem *apple_sleep_apcie_block(struct device *dev, struct device_node *np, + const char *name, const char *prop, + struct apple_tunable **tunable) { int idx = of_property_match_string(np, "reg-names", name); if (idx < 0) return NULL; if (prop) { - *tunable = apple_sleep_apcie_tunable(s, np, name, prop); + *tunable = apple_sleep_apcie_tunable(dev, np, name, prop); if (!*tunable) return NULL; } @@ -3947,18 +3469,18 @@ static void apple_sleep_map_apcie(struct apple_sleep *s) if (!np) return; - s->apcie_axi = apple_sleep_apcie_block(s, np, "axi2af", "apple,tunable-axi2af", + s->apcie_axi = apple_sleep_apcie_block(s->dev, np, "axi2af", "apple,tunable-axi2af", &s->apcie_axi_tunable); - s->apcie_cio3pll = apple_sleep_apcie_block(s, np, "cio3pll", "apple,tunable-cio3pll", + s->apcie_cio3pll = apple_sleep_apcie_block(s->dev, np, "cio3pll", "apple,tunable-cio3pll", &s->apcie_cio3pll_tunable); - s->apcie_clkgen = apple_sleep_apcie_block(s, np, "clkgen", "apple,tunable-clkgen", + s->apcie_clkgen = apple_sleep_apcie_block(s->dev, np, "clkgen", "apple,tunable-clkgen", &s->apcie_clkgen_tunable); - s->apcie_phy = apple_sleep_apcie_block(s, np, "phy-block", "apple,tunable-phy", + s->apcie_phy = apple_sleep_apcie_block(s->dev, np, "phy-block", "apple,tunable-phy", &s->apcie_phy_tunable); - s->apcie_rc = apple_sleep_apcie_block(s, np, "rc", NULL, NULL); - s->apcie_phy_ip = apple_sleep_apcie_block(s, np, "phy-ip", "apple,tunable-phy-ip-pll", + s->apcie_rc = apple_sleep_apcie_block(s->dev, np, "rc", NULL, NULL); + s->apcie_phy_ip = apple_sleep_apcie_block(s->dev, np, "phy-ip", "apple,tunable-phy-ip-pll", &s->apcie_phy_ip_pll); - s->apcie_phy_ip_lanes = apple_sleep_apcie_tunable(s, np, "phy-ip", + s->apcie_phy_ip_lanes = apple_sleep_apcie_tunable(s->dev, np, "phy-ip", "apple,tunable-phy-ip-auspma"); if (!s->apcie_axi || !s->apcie_cio3pll || !s->apcie_clkgen || !s->apcie_phy || !s->apcie_rc || !s->apcie_phy_ip || !s->apcie_phy_ip_lanes) { @@ -3970,12 +3492,12 @@ static void apple_sleep_map_apcie(struct apple_sleep *s) s->apcie_axi = NULL; } - s->apcie_port = apple_sleep_apcie_block(s, np, "port0", NULL, NULL); - s->apcie_port_phy = apple_sleep_apcie_block(s, np, "phy0", NULL, NULL); + s->apcie_port = apple_sleep_apcie_block(s->dev, np, "port0", NULL, NULL); + s->apcie_port_phy = apple_sleep_apcie_block(s->dev, np, "phy0", NULL, NULL); engine = of_parse_phandle(np, "apple,piodma", 0); if (engine) { - s->apcie_engine = apple_sleep_apcie_block(s, engine, "engine", NULL, NULL); - s->apcie_fabric = apple_sleep_apcie_block(s, engine, "fabric", NULL, NULL); + s->apcie_engine = apple_sleep_apcie_block(s->dev, engine, "engine", NULL, NULL); + s->apcie_fabric = apple_sleep_apcie_block(s->dev, engine, "fabric", NULL, NULL); of_node_put(engine); } of_node_put(np); @@ -4090,9 +3612,22 @@ static int apple_sleep_alloc_books(struct apple_sleep *s) static int apple_sleep_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + const struct apple_sleep_soc *soc = of_device_get_match_data(dev); struct apple_sleep *s; int ret; + /* A SoC with its own sequences also has its own conditions, and its own answers. */ + if (soc && soc->part) { + ret = apple_sleep_registry_bind(soc->part->registry_flags); + if (ret) + return dev_err_probe(dev, ret, "another sleep controller is bound\n"); + ret = soc->part->probe(pdev); + if (ret) + apple_sleep_registry_unbind(); + + return ret; + } + /* Below EL2 a monitor owns all of this. */ if (!is_kernel_in_hyp_mode()) return -ENODEV; @@ -4146,12 +3681,19 @@ static int apple_sleep_probe(struct platform_device *pdev) platform_set_drvdata(pdev, s); - ret = spin_table_set_power_ops(&sleep_cpu_power_ops); + ret = apple_sleep_registry_bind(APPLE_SLEEP_REG_LID_AT_WAKE | + APPLE_SLEEP_REG_USB_WAKE_PORTS); if (ret) + return dev_err_probe(dev, ret, "another sleep controller is bound\n"); + ret = spin_table_set_power_ops(&sleep_cpu_power_ops); + if (ret) { + apple_sleep_registry_unbind(); return dev_err_probe(dev, ret, "cannot control CPU power\n"); + } /* Last: the exported helpers take it as a probed driver, and a failure frees s. */ apple_sleep = s; + apple_sleep_registry_ready(); /* Deep becomes the default sleep state, unless mem_sleep_default= chose another. */ suspend_set_ops(&apple_sleep_ops); @@ -4579,6 +4121,12 @@ static const struct apple_sleep_soc t8132_sleep_soc = { .nwake_names = ARRAY_SIZE(t8132_wake_names), .wake_last_word = GENMASK(20, 0), }; +#if APPLE_SLEEP_T6040 +/* T6040 (MacBook Pro J616s): its own sequences, in sleep-t6040.c. */ +static const struct apple_sleep_soc t6040_sleep_soc = { + .part = &t6040_sleep_part, +}; +#endif /* * Only SoCs whose sleep was measured and tested. On every other the driver doesn't bind, and @@ -4587,13 +4135,31 @@ static const struct apple_sleep_soc t8132_sleep_soc = { static const struct of_device_id apple_sleep_of_match[] = { { .compatible = "apple,t8140-sleep", .data = &t8140_sleep_soc }, { .compatible = "apple,t8132-sleep", .data = &t8132_sleep_soc }, +#if APPLE_SLEEP_T6040 + { .compatible = "apple,t6040-sleep", .data = &t6040_sleep_soc }, +#endif { } }; +static int apple_sleep_restore(struct device *dev) +{ + const struct apple_sleep_soc *soc = of_device_get_match_data(dev); + + if (soc && soc->part && soc->part->restore) + return soc->part->restore(dev); + + return 0; +} + +static const struct dev_pm_ops apple_sleep_pm_ops = { + .restore = pm_sleep_ptr(apple_sleep_restore), +}; + static struct platform_driver apple_sleep_driver = { .driver = { .name = "apple-sleep", .of_match_table = apple_sleep_of_match, + .pm = pm_sleep_ptr(&apple_sleep_pm_ops), .suppress_bind_attrs = true, }, .probe = apple_sleep_probe, diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index ea0b6fe76b34a..9042ebcba7a8f 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -532,6 +532,15 @@ static void apple_spmi_irq_handler(struct irq_desc *desc) chained_irq_exit(chip, desc); } +/* Every interrupt masked, the notification included, and what is latched dropped. */ +static void apple_spmi_irq_reset(struct apple_spmi *spmi) +{ + for (size_t offset = 0; offset < (spmi->hw->num_irqs + 32) / 8; offset += 4) { + writel(0, spmi->regs + SPMI_IRQ_MASK_BASE + offset); + writel(U32_MAX, spmi->regs + SPMI_IRQ_ACK_BASE + offset); + } +} + static void apple_spmi_teardown_irq(void *data) { struct apple_spmi *spmi = data; @@ -556,10 +565,7 @@ static int apple_spmi_init_irq(struct platform_device *pdev, raw_spin_lock_init(&spmi->irq_mask_lock); - for (size_t offset = 0; offset < (spmi->hw->num_irqs + 32) / 8; offset += 4) { - writel(0, spmi->regs + SPMI_IRQ_MASK_BASE + offset); - writel(U32_MAX, spmi->regs + SPMI_IRQ_ACK_BASE + offset); - } + apple_spmi_irq_reset(spmi); spmi->irqd = devm_irq_domain_instantiate(&pdev->dev, &info); if (IS_ERR(spmi->irqd)) @@ -614,6 +620,23 @@ static int apple_spmi_resume_noirq(struct device *dev) return 0; guard(mutex)(&spmi->fifo_lock); + /* + * Firmware and the platform's sleep code use this queue while Linux is + * away. Every command of ours was answered before the suspend, so any + * reply waiting now is someone else's, and would be read as the reply + * to our next command. + */ + if (pm_resume_via_firmware()) { + unsigned int stale = 0; + + while (!(readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) && + stale < 64) { + readl(spmi->regs + spmi->hw->reply); + stale++; + } + if (stale) + dev_info(dev, "dropped %u stale reply words after the wake\n", stale); + } raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); /* Whether the enables survive a real sleep has not been measured. */ if (pm_resume_via_firmware()) { @@ -630,9 +653,6 @@ static int apple_spmi_resume_noirq(struct device *dev) return 0; } -static DEFINE_NOIRQ_DEV_PM_OPS(apple_spmi_pm_ops, apple_spmi_suspend_noirq, - apple_spmi_resume_noirq); - static int apple_spmi_probe(struct platform_device *pdev) { struct apple_spmi *spmi; @@ -688,6 +708,73 @@ static int apple_spmi_probe(struct platform_device *pdev) return 0; } +/* + * After a resume from disk the controller holds the boot kernel's masks and + * pending interrupts, not this kernel's: drop the pending ones and rewrite + * the masks from the cache the mask/unmask paths keep, notification IRQ + * included. + */ +static void apple_spmi_hib_resync_irqs(struct apple_spmi *spmi) +{ + u32 nirq = spmi->hw->num_irqs; + unsigned long flags; + + raw_spin_lock_irqsave(&spmi->irq_mask_lock, flags); + /* + * As at probe, drop what is latched: interrupts the boot kernel left + * pending complete a register selection of a device restored now + * before its own answer arrives. The devices read their state afresh. + */ + apple_spmi_irq_reset(spmi); + for (size_t offset = 0; offset < (nirq + 32) / 8; offset += 4) { + u32 val = 0; + + if (offset < nirq / 8) + val = bitmap_read(spmi->irq_mask_cache, offset * 8, 32); + else if (spmi->notify_irq && offset == (nirq / 32) * 4) + val = BIT(nirq % 32); + writel(val, spmi->regs + SPMI_IRQ_MASK_BASE + offset); + } + raw_spin_unlock_irqrestore(&spmi->irq_mask_lock, flags); +} + +/* + * Left alone, the masks stay as the boot kernel left them, everything masked + * but the interrupts of the drivers it loaded, and the IRQ core does not write + * them again (it disables lazily and unmasks only an IRQ it recorded as + * masked). A device whose driver the boot kernel did not load, such as a + * Type-C port controller on nub_spmi_a0/a1, would raise no event after the + * restore. + * + * In the early restore, for every controller alike: one in a PM domain gets no + * restore_noirq of its own, as genpd calls the driver's resume_noirq there + * (outside a wakeup path), and genpd does not wrap the early restore. It still + * runs before the devices on the bus are restored. + */ +static int apple_spmi_restore_early(struct device *dev) +{ + struct apple_spmi *spmi = dev_get_drvdata(dev); + + if (spmi->irq < 0) + return 0; + + /* Commands can run by now: none may have its reply acknowledged away. */ + scoped_guard(mutex, &spmi->fifo_lock) + apple_spmi_hib_resync_irqs(spmi); + dev_dbg(dev, "resume from disk: interrupt masks rewritten in the early restore\n"); + + return 0; +} + +/* + * Suspend-to-RAM parks the notification queue; a resume from disk rewrites the + * masks in the early restore. + */ +static const struct dev_pm_ops apple_spmi_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_spmi_suspend_noirq, apple_spmi_resume_noirq) + .restore_early = pm_sleep_ptr(apple_spmi_restore_early), +}; + static const struct of_device_id apple_spmi_match_table[] = { { .compatible = "apple,t8142-spmi", .data = &apple_spmi_gen4 }, { .compatible = "apple,t8103-spmi", .data = &apple_spmi_gen1 }, diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index 530809696b254..da51c9be40efb 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -120,8 +120,14 @@ struct apple_smc { bool sleep_warned; /* To be told at the noirq suspend, after the debug link's cut. */ bool sleep_warn_late; + /* A sleep was committed, and the wake after it has not been declared a full one. */ + bool wake_silent; + /* This SoC's SMC is told of a sleep and of a full wake through WKTP as well. */ + bool wktp; /* In a noirq phase: replies are polled for. */ bool polled; + /* Registered with the sleep controller, whose last stops can send the sleep commit. */ + bool commit_at_stop; /* MBSW at the last wake: what the SMC had woken the AP for. */ u64 wake_record; /* MBSW's bits as this SoC numbers them, from the probe. */ @@ -299,6 +305,18 @@ static inline int macsmc_hwmon_fans_request(struct device *dev, unsigned int per } #endif +/* + * Request the VDD_CIO rail (SMC key pmVC) for a system sleep: see drivers/mfd/macsmc.c. + */ +#if IS_REACHABLE(CONFIG_MFD_MACSMC) +int apple_smc_request_pmvc(struct apple_smc *smc); +#else +static inline int apple_smc_request_pmvc(struct apple_smc *smc) +{ + return -ENODEV; +} +#endif + static inline int apple_smc_read_flag(struct apple_smc *smc, smc_key key, bool *flag) { u8 val; diff --git a/include/linux/soc/apple/aura-hibernate.h b/include/linux/soc/apple/aura-hibernate.h new file mode 100644 index 0000000000000..729ced7b3c4cb --- /dev/null +++ b/include/linux/soc/apple/aura-hibernate.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __LINUX_SOC_APPLE_AURA_HIBERNATE_H +#define __LINUX_SOC_APPLE_AURA_HIBERNATE_H + +#include +#include + +struct device; +struct device_node; + +/* A firmware memory-region of a device as this boot's device tree gives it. */ +struct aura_fw_region { + u64 phys; + u64 iova; + u64 size; +}; + +/* + * Declared unconditionally so that Rust drivers can bind to them; without + * CONFIG_APPLE_AURA_HIBERNATE they must not call them. + */ +int aura_hibernate_boot_fw_regions(struct device_node *np, struct aura_fw_region *out, + unsigned int max); +int aura_hibernate_reprobe_after_thaw(struct device *dev); + +#ifdef CONFIG_APPLE_AURA_HIBERNATE +int aura_hibernate_boot_chosen_u64(const char *name, u64 *val); +#else +static inline int aura_hibernate_boot_chosen_u64(const char *name, u64 *val) { return -ENODEV; } +#endif + +#endif diff --git a/include/linux/soc/apple/rtkit-helper.h b/include/linux/soc/apple/rtkit-helper.h index 190caf432e098..4c712936e6c15 100644 --- a/include/linux/soc/apple/rtkit-helper.h +++ b/include/linux/soc/apple/rtkit-helper.h @@ -9,4 +9,20 @@ struct device; /* The caller holds a device link to the bound helper supplier. */ bool apple_rtkit_helper_preserves_input(struct device *dev); +/* + * After a resume from disk: the helper took over, in its .restore callback, + * the session that the boot kernel started and left running. Valid from the + * helper's .restore until its next .freeze. + */ +bool apple_rtkit_helper_readopted(struct device *dev); + +/* + * After a resume from disk: the session this helper had started itself was + * found with its co-processor halted (the boot loader stopped it and the boot + * kernel did not start it). Nothing answers until the helper is probed again, + * which starts the co-processor. Valid from the helper's .restore until its + * next .freeze. + */ +bool apple_rtkit_helper_halted_underneath(struct device *dev); + #endif diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 244e60bbc6989..44514f2c32eaa 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -264,6 +265,13 @@ int apple_rtkit_poweroff(struct apple_rtkit *rtk); */ int apple_rtkit_cold_restart(struct apple_rtkit *rtk, int (*start)(void *), void *ctx); +/* + * Returned by apple_rtkit_idle() and apple_rtkit_idle_with() when the AP side is idle + * and the IOP acknowledged a power state other than idle: it is asleep with its memory + * kept. No other error carries this value. + */ +#define APPLE_RTKIT_IDLE_RETAINED (-EALREADY) + /* * Put the co-processor into idle mode */ @@ -310,6 +318,16 @@ int apple_rtkit_ap_power(struct apple_rtkit *rtk, bool on); /* The same, for a noirq phase: device interrupts are off, so it polls. */ int apple_rtkit_ap_power_noirq(struct apple_rtkit *rtk, bool on); +/* + * The idle notice with interrupts off and one CPU up, where nothing can sleep and the + * receive work doesn't run: the mailbox is polled a bounded number of times, with no + * clock read, and the acknowledgment is taken in the poll. A message on one of the + * client's own endpoints still goes to recv_message_early; one it leaves waits for the + * receive work. @us is about how long the acknowledgment took. The wake's notice is + * apple_rtkit_ap_power() or apple_rtkit_ap_power_noirq(), as after any other. + */ +int apple_rtkit_ap_idle_polled(struct apple_rtkit *rtk, unsigned int *us); + /* * Stop a co-processor for a system sleep that may remove its power: idle, wait * for the status that shows it has finished, then clear the run bit. On an error @@ -442,4 +460,60 @@ void apple_rtkit_flush_rx(struct apple_rtkit *rtk); */ bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep); +/* + * A shared buffer of a running session, as the co-processor addresses it. + * @kind is one of APPLE_RTKIT_BUF_*. + */ +#define APPLE_RTKIT_BUF_IOREPORT 0 +#define APPLE_RTKIT_BUF_COREANALYTICS 1 +#define APPLE_RTKIT_BUF_CRASHLOG 2 +#define APPLE_RTKIT_BUF_OSLOG 3 +#define APPLE_RTKIT_BUF_KDEBUG0 4 +#define APPLE_RTKIT_BUF_KDEBUG1 5 +#define APPLE_RTKIT_BUF_PMP_AUX 6 +#define APPLE_RTKIT_BUF_SYSLOG 7 +#define APPLE_RTKIT_BUF_COUNT 8 + +struct apple_rtkit_buffer_desc { + u32 kind; + u32 is_mapped; + u64 iova; + u64 size; +}; + +/* + * Describe the shared buffers the co-processor currently holds, in a fixed + * order. Returns the number of entries written, or -ENOSPC if @max is too + * small. + */ +int apple_rtkit_describe_buffers(struct apple_rtkit *rtk, + struct apple_rtkit_buffer_desc *desc, + unsigned int max); + +/* + * Take over, in place, a firmware instance that another kernel started and + * left running with its shared buffers at the IOVAs this session already + * holds (the caller must have checked that with apple_rtkit_describe_buffers() + * against the other kernel's description). Only host-side transient state is + * reset; no message is sent to the co-processor. + */ +int apple_rtkit_readopt(struct apple_rtkit *rtk); + +/* + * Resume from disk. The kernel that loads a hibernation image (the boot + * kernel) can leave a few bytes for the kernel restored from that image in + * memory that the image does not overwrite. Only available with RTKit + * built in; elsewhere put() fails and take() finds nothing. + * + * apple_rtkit_hib_restoring(): true in the boot kernel between + * PM_RESTORE_PREPARE and the jump into the image (or PM_POST_RESTORE), + * i.e. while its devices are quiesced for the image, not frozen for one. + */ +#define APPLE_RTKIT_HIB_HANDOFF_MAX 256 + +bool apple_rtkit_hib_restoring(void); +int apple_rtkit_hib_handoff_put(const char *tag, const void *data, size_t len); +int apple_rtkit_hib_handoff_take(const char *tag, void *data, size_t len); +void apple_rtkit_hib_handoff_drop(const char *tag); + #endif /* _LINUX_APPLE_RTKIT_H_ */ diff --git a/include/linux/soc/apple/sleep.h b/include/linux/soc/apple/sleep.h index 07e03087cbc3d..ddd15c742e2de 100644 --- a/include/linux/soc/apple/sleep.h +++ b/include/linux/soc/apple/sleep.h @@ -9,11 +9,22 @@ #include #include #include +#include #include struct device; struct device_node; +/* + * The coprocessor and enclave registry, shared by the sleep controllers + * (drivers/soc/apple/sleep-registry.c). The controller bound on this SoC + * answers apple_sleep_commit_stops_coprocs() and runs the last stops. + */ +/* Order of the coprocessors registered first (apple_sleep_coproc_register_first()). */ +#define APPLE_SLEEP_FIRST_SMC 0 /* its warning and commit come before the GPU's stop */ +#define APPLE_SLEEP_FIRST_GPU 1 +#define APPLE_SLEEP_FIRST_ORDERS 2 + #if IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) /* * A coprocessor that can't be stopped until a sleep is certain, and can't be @@ -30,17 +41,13 @@ int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, bool pcie /* * The same with no ASC to clear: @last_stop clears its own run bits and lowers its own power * domain, and runs before every other coprocessor's. The refusal checks leave that domain on. + * Those registered first run in the order of @order, an APPLE_SLEEP_FIRST_* value, not of + * their place in the table. */ -int apple_sleep_coproc_register_first(struct device *dev, int (*last_stop)(void *data), - void *data); +int apple_sleep_coproc_register_first(struct device *dev, unsigned int order, + int (*last_stop)(void *data), void *data); void apple_sleep_coproc_unregister(struct device *dev); -/* - * During a suspend: a driver left something the DRAM-kept wake would stall on, so this - * sleep takes the deep path. False if no sleep driver can do that. - */ -bool apple_sleep_request_deep_once(void); - /* * For a registered coprocessor: the last of its stop that may still refuse the sleep, run * with interrupts off before any coprocessor's @last_stop. A negative return refuses, and the @@ -69,6 +76,61 @@ bool apple_sleep_coproc_stopped(struct device *dev); * registered refuses the sleep: stopped any earlier, a later refusal would strand it. */ bool apple_sleep_commit_stops_coprocs(void); +#else +static inline int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, + bool pcie_ans, int (*last_stop)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_coproc_register_first(struct device *dev, unsigned int order, + int (*last_stop)(void *data), void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_coproc_unregister(struct device *dev) +{ +} + +static inline int apple_sleep_coproc_set_last_check(struct device *dev, + int (*last_check)(void *data)) +{ + return -EOPNOTSUPP; +} + +static inline int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), + void *data) +{ + return -EOPNOTSUPP; +} + +static inline void apple_sleep_enclave_unregister(struct device *dev) +{ +} + +static inline void apple_sleep_coproc_set_ready(struct device *dev, bool ready) +{ +} + +static inline bool apple_sleep_coproc_stopped(struct device *dev) +{ + return false; +} + +static inline bool apple_sleep_commit_stops_coprocs(void) +{ + return false; +} +#endif + +#if IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) +/* + * During a suspend: a driver left something the DRAM-kept wake would stall on, so this + * sleep takes the deep path. False if no sleep driver can do that. + */ +bool apple_sleep_request_deep_once(void); /* * The USB host sleeps in place for this sleep, as macOS's does: its always-on domain stays @@ -117,7 +179,6 @@ bool apple_sleep_debug_link_cut_peek(struct device *dev); bool apple_sleep_debug_link_cut(void); /* This sleep will end the debug link: for the SMC's driver, which warns the SMC after it. */ -bool apple_sleep_debug_link_up(void); /* * From the SMC's driver, in its noirq resume: the SMC woke the AP for the lid alone, and the @@ -171,58 +232,12 @@ void apple_sleep_restore_pins(void); */ int apple_sleep_engine_reset(struct device *dev); #else -static inline int apple_sleep_coproc_register(struct device *dev, void __iomem *asc, - bool pcie_ans, int (*last_stop)(void *data), - void *data) -{ - return -EOPNOTSUPP; -} - -static inline int apple_sleep_coproc_register_first(struct device *dev, - int (*last_stop)(void *data), void *data) -{ - return -EOPNOTSUPP; -} - +/* Without the T8140 controller no sleep goes deep: false, as with no sleep driver. */ static inline bool apple_sleep_request_deep_once(void) { return false; } -static inline void apple_sleep_coproc_unregister(struct device *dev) -{ -} - -static inline int apple_sleep_coproc_set_last_check(struct device *dev, - int (*last_check)(void *data)) -{ - return -EOPNOTSUPP; -} - -static inline int apple_sleep_enclave_register(struct device *dev, int (*tell)(void *data), - void *data) -{ - return -EOPNOTSUPP; -} - -static inline void apple_sleep_enclave_unregister(struct device *dev) -{ -} - -static inline void apple_sleep_coproc_set_ready(struct device *dev, bool ready) -{ -} - -static inline bool apple_sleep_coproc_stopped(struct device *dev) -{ - return false; -} - -static inline bool apple_sleep_commit_stops_coprocs(void) -{ - return false; -} - static inline void apple_sleep_usb_set_asleep(bool asleep) { } @@ -270,11 +285,6 @@ static inline bool apple_sleep_debug_link_cut(void) return false; } -static inline bool apple_sleep_debug_link_up(void) -{ - return false; -} - static inline void apple_sleep_lid_closed_wake(void) { } @@ -319,4 +329,22 @@ static inline int apple_sleep_engine_reset(struct device *dev) } #endif +/* + * Whether this firmware-driven suspend may end in a real power loss of the SoC, so that a + * driver may stop what only a power cycle brings back. A dry run, which the T6040 has, sets + * pm_suspend_via_firmware() to run every sleep step that can still be taken back and then + * refuses before the commit: there this is false, and such a stop has to be left out. The + * SMC's sleep commit is one of them: an SMC that took it stays in its sleep state until the + * SoC has lost its power and come back. + * Everywhere else it is pm_suspend_via_firmware(). + */ +#if IS_ENABLED(CONFIG_APPLE_SOC_SLEEP) +bool apple_sleep_power_loss(void); +#else +static inline bool apple_sleep_power_loss(void) +{ + return pm_suspend_via_firmware(); +} +#endif + #endif diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 365437ce2cc02..f76d9d4a24cdb 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -422,6 +422,7 @@ extern void hibernation_set_ops(const struct platform_hibernation_ops *ops); extern int hibernate(void); extern bool system_entering_hibernation(void); extern bool hibernation_available(void); +void hibernation_set_platform_check(bool (*blocked)(void), const char *reason); asmlinkage int swsusp_save(void); extern struct pbe *restore_pblist; int pfn_is_nosave(unsigned long pfn); @@ -431,6 +432,23 @@ int hibernate_resume_nonboot_cpu_disable(void); int arch_hibernation_header_save(void *addr, unsigned int max_size); int arch_hibernation_header_restore(void *addr); +#ifdef CONFIG_ARCH_HIBERNATION_CLEAN_IMAGE +/** + * arch_hibernate_clean_image_page - Make a loaded image page visible to devices. + * @pfn: Valid page frame of the image. + * + * Provided by an architecture that sets ARCH_HIBERNATION_CLEAN_IMAGE. + * Called once for every valid page frame of the image after the whole image + * has been loaded, while the list of those frames is still intact, so that + * the architecture can clean the caches for devices that do not snoop them. + * + * Return: 1 when the page was cleaned, 0 when it was left alone because + * nothing was loaded into it through the linear map (the frame is restored + * from restore_pblist). + */ +int arch_hibernate_clean_image_page(unsigned long pfn); +#endif + #else /* CONFIG_HIBERNATION */ static inline void register_nosave_region(unsigned long b, unsigned long e) {} static inline int swsusp_page_is_forbidden(struct page *p) { return 0; } @@ -453,6 +471,16 @@ bool pm_hibernation_mode_is_suspend(void); static inline bool pm_hibernation_mode_is_suspend(void) { return false; } #endif +#ifdef CONFIG_HIBERNATION +bool pm_hibernate_is_restored(void); +bool pm_hibernate_is_loading(void); +int pm_hibernate_restored_cold(void); +#else +static inline bool pm_hibernate_is_restored(void) { return false; } +static inline bool pm_hibernate_is_loading(void) { return false; } +static inline int pm_hibernate_restored_cold(void) { return 0; } +#endif + int arch_resume_nosmt(void); #ifdef CONFIG_HIBERNATION_SNAPSHOT_DEV diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 890bf7b447f89..f771766758f0f 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include "power.h" @@ -81,6 +80,105 @@ static const struct platform_hibernation_ops *hibernation_ops; static atomic_t hibernate_atomic = ATOMIC_INIT(1); +/* Set when swsusp_arch_suspend() returned in a kernel restored from an image. */ +static bool hibernate_restored; + +/** + * pm_hibernate_is_restored - Whether this kernel was just restored from an image. + * + * Devices get PMSG_RESTORE in the kernel restored from a hibernation image, + * whose hardware went through a boot, and PMSG_RECOVER in the kernel that + * hibernated when hibernation is aborted before the image is created (a device + * failed to freeze, a wakeup event). True from the restore until the next + * hibernation starts (hibernation_snapshot()); meant for code that runs after + * the restore, such as the PM_POST_HIBERNATION notifiers. + */ +bool pm_hibernate_is_restored(void) +{ + return READ_ONCE(hibernate_restored); +} +EXPORT_SYMBOL_GPL(pm_hibernate_is_restored); + +/* + * What the kernel that loads an image leaves for the kernel restored from it. + * It lives in the no-save section, which the image neither saves nor + * overwrites, so the restored kernel reads what the loading kernel wrote, as + * long as both are the same build loaded at the same physical address (which + * the image's header requires). The record carries its own physical address + * and a magic: a record from a kernel loaded elsewhere, or none, reads as + * absent. + */ +struct hibernate_load_record { + u32 magic; + u32 other_boot; + u64 self_phys; +}; + +#define HIBERNATE_LOAD_MAGIC 0x484c4452 /* "HLDR" */ + +static struct hibernate_load_record hibernate_load_record __nosavedata; + +/* Set while hibernate() loads the image it has just written (test_resume). */ +static bool hibernate_loading_own_image; + +/* Whether the restore came from another boot, see pm_hibernate_restored_cold(). */ +static int hibernate_restored_cold = -ENODATA; + +/** + * pm_hibernate_restored_cold - Whether this kernel was restored by another boot. + * + * Meaningful once pm_hibernate_is_restored() is true, under the same lifetime. + * The kernel that loads an image tells the kernel restored from it whether it + * is the kernel that wrote the image (a test_resume load: the machine did not + * boot) or one that was started to resume it (the boot loader and the + * firmware have been through a boot). Whether a particular device lost its + * state across that boot is for its driver to find out. + * + * Return: 1 if the image was loaded by another boot, 0 if this kernel loaded + * its own image or was not restored, -ENODATA if the record of the loading + * kernel is missing. + */ +int pm_hibernate_restored_cold(void) +{ + if (!READ_ONCE(hibernate_restored)) + return 0; + return READ_ONCE(hibernate_restored_cold); +} +EXPORT_SYMBOL_GPL(pm_hibernate_restored_cold); + +/* Set while this kernel loads a hibernation image, see pm_hibernate_is_loading(). */ +static bool hibernate_loading; + +/** + * pm_hibernate_is_loading - Whether this kernel is loading a hibernation image. + * + * True from the PM_RESTORE_PREPARE notification, which starts the loading of an + * image (from the resume path or through the snapshot device), until the + * PM_POST_RESTORE notification that ends it when the image was not restored + * (or the notification chain refused it). The devices of the kernel that + * loads an image are quiesced for the jump into the image, not frozen for an + * image of their own; the flag lets drivers tell the two apart in their + * freeze callbacks. The kernel restored from an image reads false: the image + * holds the state of the kernel that created it. + */ +bool pm_hibernate_is_loading(void) +{ + return READ_ONCE(hibernate_loading); +} +EXPORT_SYMBOL_GPL(pm_hibernate_is_loading); + +/** + * hibernate_set_loading - Record that an image is being loaded or no longer is. + * @loading: New value of the flag read by pm_hibernate_is_loading(). + * + * Called from the PM notifier chain helpers around PM_RESTORE_PREPARE and + * PM_POST_RESTORE. + */ +void hibernate_set_loading(bool loading) +{ + WRITE_ONCE(hibernate_loading, loading); +} + #ifdef CONFIG_SUSPEND /** * pm_hibernation_mode_is_suspend - Check if hibernation has been set to suspend @@ -107,23 +205,36 @@ bool hibernation_in_progress(void) return !atomic_read(&hibernate_atomic); } -/* - * Apple silicon Macs run firmware on their coprocessors (storage, secure - * enclave, display, always-on processor, system controller, GPU and more). - * Restoring an image means a cold boot through the platform boot chain, which - * restarts that firmware, but most of the Linux drivers have no - * freeze/poweroff/restore path that rebuilds their mailbox, ring and DMA state - * from a cold start. The restored kernel would keep using stale state, so - * writing and restoring images stays disabled on these machines. +static bool (*hibernation_platform_blocked_fn)(void); +static const char *hibernation_platform_blocked_reason; + +/** + * hibernation_set_platform_check - Let platform code refuse hibernation. + * @blocked: Tells whether hibernation is refused on this platform at the time. + * @reason: Why it is refused, for the log. + * + * A platform whose drivers cannot yet restore their devices from an image + * registers a check that is true until the platform has what it needs. While + * it is true, writing and restoring images stays disabled + * (hibernation_available() is false) and @reason is logged at the end of the + * boot. Only one check can be registered. * * pm_test dry runs are allowed (see hibernate()): every pm_test level stops * before create_image() takes a snapshot, so no image is written or restored. * They exercise the freezer and the device freeze/thaw callbacks that a real * implementation needs. */ +void hibernation_set_platform_check(bool (*blocked)(void), const char *reason) +{ + if (WARN_ON_ONCE(hibernation_platform_blocked_fn)) + return; + hibernation_platform_blocked_reason = reason; + hibernation_platform_blocked_fn = blocked; +} + static bool hibernation_platform_blocked(void) { - return of_machine_is_compatible("apple,arm-platform"); + return hibernation_platform_blocked_fn && hibernation_platform_blocked_fn(); } static bool hibernation_generic_available(void) @@ -142,7 +253,7 @@ bool hibernation_available(void) static int __init hibernation_platform_report(void) { if (hibernation_platform_blocked()) - pr_info("disabled on this platform: coprocessor firmware state is not restored from an image\n"); + pr_info("disabled on this platform: %s\n", hibernation_platform_blocked_reason); return 0; } late_initcall(hibernation_platform_report); @@ -346,6 +457,25 @@ __weak int arch_resume_nosmt(void) return 0; } +/** + * hibernate_resume_event - Pick the event that resumes the devices frozen for an image. + * @error: Result of the image creation so far. + * + * Called once the image is created, restored or given up on. in_suspend is + * set only in the kernel that created the image, so a clear one means either + * that this kernel was restored from an image (devices get PMSG_RESTORE, + * their hardware went through a boot) or that the hibernation was aborted + * before the image existed (a device failed to freeze, a wakeup event, a + * pm_test level): the devices were only frozen, and the right event is + * PMSG_RECOVER, as after any other failed freeze. + */ +static pm_message_t hibernate_resume_event(int error) +{ + if (in_suspend) + return error ? PMSG_RECOVER : PMSG_THAW; + return pm_hibernate_is_restored() ? PMSG_RESTORE : PMSG_RECOVER; +} + /** * create_image - Create a hibernation image. * @platform_mode: Whether or not to use the platform driver. @@ -399,6 +529,13 @@ static int create_image(int platform_mode) if (!in_suspend) { events_check_enabled = false; clear_or_poison_free_pages(); + if (hibernate_load_record.magic == HIBERNATE_LOAD_MAGIC && + hibernate_load_record.self_phys == __pa_symbol(&hibernate_load_record)) + WRITE_ONCE(hibernate_restored_cold, !!hibernate_load_record.other_boot); + else + WRITE_ONCE(hibernate_restored_cold, -ENODATA); + hibernate_load_record.magic = 0; + WRITE_ONCE(hibernate_restored, true); } platform_leave(platform_mode); @@ -420,8 +557,7 @@ static int create_image(int platform_mode) Platform_finish: platform_finish(platform_mode); - dpm_resume_start(in_suspend ? - (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE); + dpm_resume_start(hibernate_resume_event(error)); return error; } @@ -438,6 +574,8 @@ int hibernation_snapshot(int platform_mode) int error; pm_suspend_clear_flags(); + WRITE_ONCE(hibernate_restored, false); + WRITE_ONCE(hibernate_restored_cold, -ENODATA); error = platform_begin(platform_mode); if (error) goto Close; @@ -485,7 +623,7 @@ int hibernation_snapshot(int platform_mode) if (error || !in_suspend) swsusp_free(); - msg = in_suspend ? (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE; + msg = hibernate_resume_event(error); dpm_resume(msg); if (error || !in_suspend) @@ -551,6 +689,9 @@ static int resume_target_kernel(bool platform_mode) save_processor_state(); error = restore_highmem(); if (!error) { + hibernate_load_record.other_boot = !hibernate_loading_own_image; + hibernate_load_record.self_phys = __pa_symbol(&hibernate_load_record); + hibernate_load_record.magic = HIBERNATE_LOAD_MAGIC; error = swsusp_arch_resume(); /* * The code below is only ever reached in case of a failure. @@ -558,6 +699,7 @@ static int resume_target_kernel(bool platform_mode) * swsusp_arch_suspend() was called. */ BUG_ON(!error); + hibernate_load_record.magic = 0; /* * This call to restore_highmem() reverts the changes made by * the previous one. @@ -774,6 +916,18 @@ static int load_image_and_restore(void) } error = swsusp_read(&flags); + if (!error) { + /* + * The image is in memory now. Clear its on-disk signature + * before the device restore, so that a restore which hangs + * (for example a co-processor that does not come back after + * the cold boot) cannot leave a valid image for the next boot + * to resume into the same hang. swsusp_close() releases the + * resume device, so do it while that is still open. + */ + if (swsusp_invalidate_sig()) + pr_warn("could not invalidate the image signature; a failed restore may repeat\n"); + } swsusp_close(); if (!error) error = hibernation_restore(flags & SF_PLATFORM_MODE); @@ -917,8 +1071,11 @@ int hibernate(void) if (snapshot_test) { pm_pr_dbg("Checking hibernation image\n"); error = swsusp_check(false); - if (!error) + if (!error) { + hibernate_loading_own_image = true; error = load_image_and_restore(); + hibernate_loading_own_image = false; + } } thaw_processes(); diff --git a/kernel/power/main.c b/kernel/power/main.c index 5429e9f19b655..ccced400c15c7 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -169,14 +169,25 @@ int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down) { int ret; + if (val_up == PM_RESTORE_PREPARE) + hibernate_set_loading(true); + ret = blocking_notifier_call_chain_robust(&pm_chain_head, val_up, val_down, NULL); + if (ret && val_up == PM_RESTORE_PREPARE) + hibernate_set_loading(false); + return notifier_to_errno(ret); } int pm_notifier_call_chain(unsigned long val) { - return blocking_notifier_call_chain(&pm_chain_head, val, NULL); + int ret = blocking_notifier_call_chain(&pm_chain_head, val, NULL); + + if (val == PM_POST_RESTORE) + hibernate_set_loading(false); + + return ret; } /* If set, devices may be suspended and resumed asynchronously. */ diff --git a/kernel/power/power.h b/kernel/power/power.h index 75b63843886ee..e43e77f9a999b 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -77,6 +77,7 @@ static inline void enable_restore_image_protection(void) {} #endif /* CONFIG_STRICT_KERNEL_RWX */ extern bool hibernation_in_progress(void); +void hibernate_set_loading(bool loading); #else /* !CONFIG_HIBERNATION */ @@ -84,6 +85,7 @@ static inline void hibernate_reserved_size_init(void) {} static inline void hibernate_image_size_init(void) {} static inline bool hibernation_in_progress(void) { return false; } +static inline void hibernate_set_loading(bool loading) {} #endif /* !CONFIG_HIBERNATION */ #define power_attr(_name) \ @@ -201,6 +203,7 @@ extern void swsusp_free(void); extern int swsusp_read(unsigned int *flags_p); extern int swsusp_write(unsigned int flags); void swsusp_close(void); +int swsusp_invalidate_sig(void); #ifdef CONFIG_SUSPEND extern int swsusp_unmark(void); #else diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index b209712cb2c3a..42d24f43991f8 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -36,6 +36,7 @@ #include #include #include +#include #include "power.h" @@ -2750,6 +2751,37 @@ static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca) return pbe->address; } +#ifdef CONFIG_ARCH_HIBERNATION_CLEAN_IMAGE +/* + * Most image pages are loaded straight into their original frames through + * the cacheable linear map (only conflicting pages go through restore_pblist). + * Where devices read memory without snooping the CPU caches, a later eviction + * of a dirty line would overwrite what a device wrote since: let the + * architecture clean every image page before the jump. + */ +static void swsusp_clean_restored_image(void) +{ + unsigned long pfn, n = 0, skipped = 0; + + memory_bm_position_reset(&orig_bm); + for (pfn = memory_bm_next_pfn(&orig_bm); pfn != BM_END_OF_MAP; + pfn = memory_bm_next_pfn(&orig_bm)) { + if (!pfn_valid(pfn)) + continue; + if (!arch_hibernate_clean_image_page(pfn)) { + skipped++; + continue; + } + if (!(++n & 0xffff)) + cond_resched(); + } + pr_debug("%lu image pages cleaned, %lu left to the restore_pblist copy\n", + n, skipped); +} +#else +static inline void swsusp_clean_restored_image(void) {} +#endif + /** * snapshot_write_next - Get the address to store the next image page. * @handle: Snapshot handle structure to guide the writing. @@ -2871,6 +2903,8 @@ int snapshot_write_finalize(struct snapshot_handle *handle) error = hibernate_restore_protect_page(handle->buffer); /* Do that only if we have loaded the image entirely */ if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages) { + /* while orig_bm still lists the image: recycling overwrites it */ + swsusp_clean_restored_image(); memory_bm_recycle(&orig_bm); free_highmem_data(); } diff --git a/kernel/power/swap.c b/kernel/power/swap.c index c78f1593600bb..8cc439448e053 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -483,11 +483,27 @@ static int flush_swap_writer(struct swap_map_handle *handle) static int swap_writer_finish(struct swap_map_handle *handle, unsigned int flags, int error) { + int flush; + if (!error) { pr_info("S"); error = mark_swapfiles(handle, flags); pr_cont("|\n"); flush_swap_writer(handle); + /* + * Make the image durable now: otherwise the device cache is + * flushed only by the device shutdown in the power-off that + * follows, and a platform whose power-off does not get that far + * loses the image. A failed flush leaves no durable image: drop + * the signature durably (unlike swsusp_unmark()) so that no + * later boot resumes from freed swap. + */ + flush = blkdev_issue_flush(file_bdev(hib_resume_bdev_file)); + if (!error && flush) { + if (swsusp_invalidate_sig()) + pr_err("image flush failed and the signature could not be dropped\n"); + error = flush; + } } if (error) @@ -1583,8 +1599,13 @@ int swsusp_check(bool exclusive) if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) { memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); swsusp_header_flags = swsusp_header->flags; - /* Reset swap signature now */ - error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, + /* + * Reset swap signature now. FUA: a REQ_SYNC write + * can sit in a volatile device cache, and a hard + * power-off of a hung restore then loses it and the + * next boot resumes the same image again. + */ + error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC | REQ_FUA, swsusp_resume_block, swsusp_header); } else { @@ -1625,6 +1646,39 @@ void swsusp_close(void) fput(hib_resume_bdev_file); } +/* + * Clear the on-disk image signature without freeing the swap pages. + * Called on the resume path right after the image is read into memory and + * before the device restore, so a restore that hangs (a co-processor that does + * not come back after a cold boot) cannot leave a valid image that the next + * boot resumes into the same hang. Unlike swsusp_unmark() this keeps the swap + * pages: the resuming kernel is about to jump into the restored image, and + * freeing swap here would desync the restored kernel's swap accounting. + */ +int swsusp_invalidate_sig(void) +{ + int error; + + error = hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block, swsusp_header); + if (error) + return error; + /* + * swsusp_check() normally reset the signature already; that write may + * still be in the device's volatile cache, so rewrite the header with + * FUA either way and make the reset durable before the restore. + */ + if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) + memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); + else if (memcmp(swsusp_header->sig, swsusp_header->orig_sig, 10)) + return -ENODEV; + error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC | REQ_FUA, + swsusp_resume_block, swsusp_header); + /* A device that ignores FUA still honours an explicit cache flush. */ + if (!error) + error = blkdev_issue_flush(file_bdev(hib_resume_bdev_file)); + return error; +} + /** * swsusp_unmark - Unmark swsusp signature in the resume device * diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 01e0e6cfbced8..3da0006a9c8ea 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -114,6 +114,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/suspend.c b/rust/helpers/suspend.c index a616907ecc0b5..9012a7f730569 100644 --- a/rust/helpers/suspend.c +++ b/rust/helpers/suspend.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include __rust_helper bool rust_helper_pm_suspend_via_firmware(void) @@ -11,3 +12,15 @@ __rust_helper bool rust_helper_pm_resume_via_firmware(void) { return pm_resume_via_firmware(); } + +/* An inline without CONFIG_HIBERNATION, so Rust reaches it here. */ +__rust_helper bool rust_helper_pm_hibernate_is_restored(void) +{ + return pm_hibernate_is_restored(); +} + +/* An inline on most configurations, so Rust reaches it here (linux/soc/apple/sleep.h). */ +__rust_helper bool rust_helper_apple_sleep_power_loss(void) +{ + return apple_sleep_power_loss(); +} diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index c34f187f4fb10..e4527ff598126 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -367,6 +367,15 @@ impl RtKit { to_result(unsafe { bindings::apple_rtkit_wake(self.rtk) }) } + /// Re-initialises a stopped session so a fresh HELLO/EPMAP boot can be + /// accepted again (hibernation restore of a coprocessor the boot loader + /// cold-restarted): detaches RX, frees inherited buffers, zeroes the + /// endpoint bitmap, reinitialises the completions and restarts the mailbox. + pub fn reinit(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_reinit(self.rtk) }) + } + /// Waits for the RTKit coprocessor to finish booting. pub fn boot(self: Pin<&mut Self>) -> Result { // SAFETY: `rtk` is valid per the type invariant. diff --git a/scripts/dtc/apple-dt-check.sh b/scripts/dtc/apple-dt-check.sh index 04b634e886427..820d8dc2daf41 100755 --- a/scripts/dtc/apple-dt-check.sh +++ b/scripts/dtc/apple-dt-check.sh @@ -2,7 +2,9 @@ # SPDX-License-Identifier: GPL-2.0-only # apple-dt-check.sh BASE HEAD WORKDIR: run CHECK_DTBS on every Apple device tree at BASE and at HEAD # and fail when HEAD has a warning BASE does not. Also runs dt_binding_check on every binding HEAD -# changes. Phandle values in the messages are masked, so renumbering is not a new warning. +# changes. Phandle values in the messages are masked, so renumbering is not a new warning, and so +# is the pattern list of a "does not match any of the regexes" message, which grows with every +# vendor prefix the bindings add. # Needs dtschema (dt-validate), clang/lld (LLVM=1), flex, bison. Prints a summary. set -u BASE=$1 HEAD=$2 W=$3 @@ -17,7 +19,7 @@ warnings() { # rev tag local rc=$? grep -E '\.dtb: |[Ww]arning|rror' "$W/$2.log" | grep -v '^ ' | sed -E "s#$t/##g; s#$o/##g" | - perl -pe 's{(\[\[[^:]*?\]\])}{(my $x = $1) =~ s/\d+/N/g; $x}ge; s{b\x27(?:\\.|[^\x27])*\x27}{b\x27N\x27}g' | + perl -pe 's{(\[\[[^:]*?\]\])}{(my $x = $1) =~ s/\d+/N/g; $x}ge; s{b\x27(?:\\.|[^\x27])*\x27}{b\x27N\x27}g; s{( not match any of the regexes): .*}{$1: ...}' | sort -u > "$W/$2.warn" echo "$2: make rc=$rc, $(wc -l < "$W/$2.warn") warning lines, $(echo $dtbs | wc -w) DTBs" return $rc diff --git a/tools/asahi/pmu-rtc-tests.py b/tools/asahi/pmu-rtc-tests.py index 062597ca80b56..02b271bc28e12 100644 --- a/tools/asahi/pmu-rtc-tests.py +++ b/tools/asahi/pmu-rtc-tests.py @@ -28,6 +28,8 @@ 'baku-limit': ('.max_delay = U32_MAX,', '.max_delay = APPLE_PMU_RTC_MAX_DELAY,'), 'set-no-round': ('(counter + BIT(APPLE_PMU_RTC_HZ_SHIFT - 1))', '(counter)'), 'set-negative': ('if (offset < 0 || offset', 'if (offset'), + 'restore-nothing': ('rtc_update_irq(rtc->clock, 1, RTC_IRQF | RTC_AF);', + 'if (false) rtc_update_irq(rtc->clock, 1, RTC_IRQF | RTC_AF);'), } PRELUDE = r''' @@ -57,6 +59,8 @@ #define kfree free #define guard(x) (void) #define dev_err(...) ((void)0) +#define dev_warn(...) ((void)0) +#define PM_SUSPEND_ON 0 #define PM_SUSPEND_TO_IDLE 1 #define RTC_IRQF 0x80 #define RTC_AF 0x20 @@ -333,7 +337,18 @@ firmware_suspend = true; assert(!apple_pmu_rtc_prepare(&dev)); firmware_suspend = false; + /* Hibernation is accepted. A platform power-off stops the alarm, and goes on if that fails. */ + pm_suspend_target_state = PM_SUSPEND_ON; + assert(!apple_pmu_rtc_prepare(&dev)); + assert(!apple_pmu_rtc_set_alarm(&dev, &alarm) && (map.bytes[map.ctrl] & 0x40)); + assert(!apple_pmu_rtc_poweroff(&dev)); + assert(!(map.bytes[map.ctrl] & 0x40) && (map.bytes[map.mask] & 1)); + map.ops = 0; map.fail = 1; + assert(!apple_pmu_rtc_poweroff(&dev)); pm_suspend_target_state = PM_SUSPEND_TO_IDLE; + /* After a resume from disk the RTC core goes through its alarm again, without a PMU access of the driver's own. */ + reset(v); + assert(!apple_pmu_rtc_restore(&dev) && clock_model.events == 1 && map.ops == 0); } int main(void) { run_variant(BAKU); run_variant(ABBEY); run_variant(STOWE); run_variant(MAVERICK); diff --git a/tools/asahi/sep-suspend-errors-tests.py b/tools/asahi/sep-suspend-errors-tests.py index 2ae69ec2c32fc..23dc813080519 100644 --- a/tools/asahi/sep-suspend-errors-tests.py +++ b/tools/asahi/sep-suspend-errors-tests.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT """Compile actual SEP commit/error paths with host-only dependency stubs. -Extract current C stop/finish/prepare/noirq, provider poll-one and registration, -Rust wrappers/enter/leave/exit/kick, and the kernel polling macros verbatim. +Extract current C stop/finish/prepare/noirq, the PM notifier, provider poll-one and +registration, Rust wrappers/enter/leave/exit/kick, and the kernel polling macros verbatim. C fatal paths are observed wholly within C, never by unwinding a Rust frame. Virtual time/MMIO only test AP control decisions, not firmware timing or safety. No target access or kernel build occurs. @@ -88,6 +88,12 @@ def replace_once(source, before, after): 'msg = apple_mbox_read_msg(mbox);\n\t\tmbox->rx(mbox, msg, mbox->cookie);\n\t\tret = 2;'), 'post-send-wake-refusal': ('sep_pm_commit_stop', 'c->sent = true;', 'c->sent = true;\n\tif (pm_wakeup_pending()) return -EBUSY;'), + 'held-before-refusal': ('sep_pm_notify', 'if (event(SEP_PM_RESTORE) > 0) {', + 'sep_diag_sleep(true);\n\t\tif (event(SEP_PM_RESTORE) > 0) {'), + 'cold-as-warm': ('sep_pm_notify', 'if (cold) {', 'if (false) {'), + 'unrecorded-as-warm': ('sep_pm_notify', 'if (cold < 0) {', 'if (false) {'), + 'unrecorded-as-cold': ('sep_pm_notify', 'cold = pm_hibernate_restored_cold();', + 'cold = pm_hibernate_restored_cold() != 0;'), } C_PREFIX = r''' @@ -138,7 +144,12 @@ def replace_once(source, before, after): static struct apple_mbox_hw hw; static struct apple_mbox mbox; static int apple_sleep_coprocs_lock; -static void *apple_sleep; +#define APPLE_SLEEP_REG_ENCLAVE_ANY BIT(0) +#define APPLE_SLEEP_REG_BOUND BIT(8) +#define APPLE_SLEEP_REG_READY BIT(9) +/* The T8140 controller as its probe leaves the registry: bound, then ready. */ +static unsigned int apple_sleep_reg = APPLE_SLEEP_REG_BOUND | APPLE_SLEEP_REG_READY; +static unsigned int apple_sleep_dry; static struct { struct device *dev; int (*stop)(void *); void *data; } apple_sleep_enclave; @@ -154,6 +165,22 @@ def replace_once(source, before, after): static unsigned queued, consumed; static ktime_t now; static bool (*sep_pm_told)(void); +/* The PM notifier: its events go to the driver, and the diag state shows a sleep held. */ +struct notifier_block { int unused; }; +enum { NOTIFY_DONE, NOTIFY_OK, NOTIFY_BAD = 0x8002 }; +enum { PM_HIBERNATION_PREPARE = 1, PM_POST_HIBERNATION, PM_SUSPEND_PREPARE, PM_POST_SUSPEND, + PM_RESTORE_PREPARE, PM_POST_RESTORE }; +#define pr_err(...) ((void)0) +static int (*sep_pm_event)(int event); +static struct device *sep_pm_dev; +static int restored_cold, held, sleeps_ended, reprobes; +static bool restored; +static void sep_diag_sleep(bool entering) { + if (entering) held = 1; else if (held) { held = 0; ++sleeps_ended; } +} +static int pm_hibernate_restored_cold(void) { return restored_cold; } +static bool pm_hibernate_is_restored(void) { return restored; } +static void sep_pm_schedule_reprobe(struct device *d) { ++reprobes; } static bool yes(void) { return true; } static bool no(void) { return false; } static bool pm_suspend_via_firmware(void) { return firmware; } @@ -251,7 +278,7 @@ def replace_once(source, before, after): memset(&hw, 0, sizeof(hw)); mbox = (struct apple_mbox){ .regs = &hw, .hw = &hw, .rx = foreign_rx, .cookie = &device }; config_enabled = hyp = sep_pm_suspending = dart_present = dart_ready = map_ok = true; - firmware = tx_ready = true; apple_sleep = &device; + firmware = tx_ready = true; apple_sleep_reg = APPLE_SLEEP_REG_BOUND | APPLE_SLEEP_REG_READY; real_wake = pending_wake = late_wake = sleep_refuse_before_send = false; domain_error = property_error = address_error = read_error = gate_after = reads = 0; put_refs = maps = unmaps = lookups = prepares = restores = sends = closes = 0; @@ -260,13 +287,15 @@ def replace_once(source, before, after): resource_bytes = 4; close_gate = 1; early_gate = 0; read_error_after = 1; queued = consumed = 0; message_in_lock = matched_read = false; now = 0; fatal_armed = false; fatal_message[0] = 0; sep_pm_told = yes; + sep_pm_event = NULL; sep_pm_dev = &device; restored_cold = 0; + held = sleeps_ended = reprobes = 0; restored = false; } void parameter(int which, int value) { switch (which) { case 0: config_enabled = value; break; case 1: hyp = value; break; case 2: sep_pm_suspending = value; break; case 3: dart_present = value; break; case 4: map_ok = value; break; case 5: domain_error = value; break; - case 6: read_error = value; break; case 7: apple_sleep = value ? &device : NULL; break; + case 6: read_error = value; break; case 7: apple_sleep_reg = APPLE_SLEEP_REG_BOUND | (value ? APPLE_SLEEP_REG_READY : 0); break; case 8: gate_after = value; break; case 9: sep_pm_commit.registered = value; break; case 10: read_error_after = value; break; case 11: dart_ready = value; break; case 12: real_wake = value; break; case 13: hw.is_ascwrap_v8 = value; break; @@ -435,8 +464,50 @@ def replace_once(source, before, after): fixture(); sep_pm_told = NULL; assert(noirq() == 0 && !lookups); fixture(); sep_pm_told = no; assert(noirq() == 0 && !lookups); } +static int attached, last_event; +static int driver_event(int event) { + last_event = event; + // The driver reports a live session to drop for a cold boot, like the real one. + return event == SEP_PM_RESTORE ? attached : event == SEP_PM_HIBERNATE_COLD; +} +static void test_notifier(void) { + struct notifier_block nb; + // No driver registered: the chain goes on untouched. + fixture(); assert(sep_pm_notify(&nb, PM_RESTORE_PREPARE, NULL) == NOTIFY_DONE && !held); + // A kernel that attached the enclave refuses to load an image. The PM core sends no + // PM_POST_RESTORE to the notifier that refused, so nothing may be left held. + fixture(); sep_pm_event = driver_event; attached = 1; + assert(sep_pm_notify(&nb, PM_RESTORE_PREPARE, NULL) == NOTIFY_BAD); + assert(last_event == SEP_PM_RESTORE && !held && !sleeps_ended); + // One that did not goes on, and the end of the restore ends the hold. + attached = 0; + assert(sep_pm_notify(&nb, PM_RESTORE_PREPARE, NULL) == NOTIFY_OK && held); + assert(sep_pm_notify(&nb, PM_POST_RESTORE, NULL) == NOTIFY_OK); + assert(last_event == SEP_PM_HIBERNATE_WARM && !held && sleeps_ended == 1); + // The end of a hibernation: an aborted one or a hybrid sleep's wake was never restored. + fixture(); sep_pm_event = driver_event; restored_cold = 1; + assert(sep_pm_notify(&nb, PM_HIBERNATION_PREPARE, NULL) == NOTIFY_OK && held); + assert(sep_pm_notify(&nb, PM_POST_HIBERNATION, NULL) == NOTIFY_OK); + assert(last_event == SEP_PM_HIBERNATE_WARM && !held && sleeps_ended == 1 && !reprobes); + // Restored after a boot that loaded the image: the enclave restarted, the device is probed again. + restored = true; restored_cold = 1; + assert(sep_pm_notify(&nb, PM_POST_HIBERNATION, NULL) == NOTIFY_OK); + assert(last_event == SEP_PM_HIBERNATE_COLD && reprobes == 1); + // Restored by the kernel that wrote the image (test_resume): the enclave never restarted. + restored_cold = 0; reprobes = 0; + assert(sep_pm_notify(&nb, PM_POST_HIBERNATION, NULL) == NOTIFY_OK); + assert(last_event == SEP_PM_HIBERNATE_WARM && !reprobes); + // No record of the loading kernel: the session is dropped, no probe is queued. + restored_cold = -ENODATA; + assert(sep_pm_notify(&nb, PM_POST_HIBERNATION, NULL) == NOTIFY_OK); + assert(last_event == SEP_PM_HIBERNATE_COLD && !reprobes); + // A suspend holds and releases the same way. + assert(sep_pm_notify(&nb, PM_SUSPEND_PREPARE, NULL) == NOTIFY_OK && held); + assert(sep_pm_notify(&nb, PM_POST_SUSPEND, NULL) == NOTIFY_OK && !held && sleeps_ended == 2); +} int main(void) { test_prepare_and_cleanup(); test_unsent(); test_provider(); test_sent(); test_poll_contract(); + test_notifier(); puts("SEP C commit controls passed"); return 0; } #endif @@ -446,7 +517,7 @@ def replace_once(source, before, after): #![allow(dead_code, unused_imports, unused_variables)] use std::cell::{Cell, RefCell, RefMut}; use std::ffi::{c_int, c_void}; -use std::sync::atomic::{AtomicU32, Ordering, Ordering::Relaxed}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering, Ordering::Relaxed}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct Error(i32); impl Error { fn from_errno(n: i32) -> Self { assert!(n < 0); Self(n) } } type Result = std::result::Result; @@ -529,13 +600,15 @@ def replace_once(source, before, after): struct SepData { dev: kernel::device::Device, phase: AtomicU32, control: Lock, mbox: Lock>, gate: Cell, sleep_rx_held: Flag, endpoint: Cell, control_wq: Wq, + session_gone: AtomicBool, } impl SepData { fn new() -> Self { Self { dev: kernel::device::Device, phase: AtomicU32::new(PHASE_READY), control: Lock(RefCell::new(Control { busy: false, wake_ok: true })), mbox: Lock(RefCell::new(Some(Mbox))), gate: Cell::new(pm::SleepGate::Open), - sleep_rx_held: Flag(Cell::new(false)), endpoint: Cell::new(true), control_wq: Wq } + sleep_rx_held: Flag(Cell::new(false)), endpoint: Cell::new(true), control_wq: Wq, + session_gone: AtomicBool::new(false) } } fn endpoint_present(&self, _: u32) -> bool { self.endpoint.get() } fn sleep_gate(&self) -> pm::SleepGate { self.gate.get() } @@ -659,21 +732,27 @@ def sources(source, contracts, mutation): pm = (source / 'drivers/soc/apple/pm.rs').read_text() sep = (source / 'drivers/soc/apple/sep.rs').read_text() shim = (source / 'drivers/soc/apple/pm_shim.c').read_text() + events = re.findall(r'^#define SEP_PM_[A-Z_]+\s+\d+$', + (source / 'drivers/soc/apple/shim.h').read_text(), re.M) + assert len(events) == 6, events mailbox = (source / 'drivers/soc/apple/mailbox.c').read_text() sleep = (source / 'drivers/soc/apple/sleep.c').read_text() + registry = (source / 'drivers/soc/apple/sleep-registry.c').read_text() iopoll = (contracts / 'include/linux/iopoll.h').read_text() names = { 'shim': ['sep_pm_wait_gated', 'sep_pm_clear_dev_disable', 'sep_pm_dart_restore', 'sep_pm_commit_rx', 'sep_pm_commit_take_rx', 'sep_pm_commit_stop', - 'sep_pm_commit_prepare', 'sep_pm_commit_finish', 'sep_pm_suspend_noirq'], + 'sep_pm_commit_prepare', 'sep_pm_commit_finish', 'sep_pm_suspend_noirq', + 'sep_pm_notify'], 'mailbox': ['apple_mbox_poll_one_supported', 'apple_mbox_poll_one'], - 'sleep': ['apple_sleep_commit_stops_coprocs', 'apple_sleep_enclave_register', - 'apple_sleep_enclave_unregister'], + 'sleep': ['apple_sleep_power_loss'], + 'registry': ['apple_sleep_commit_stops_coprocs', 'apple_sleep_enclave_register', + 'apple_sleep_enclave_unregister'], 'pm': ['leave_for_commit', 'commit_finish', 'clear_dev_disable', 'dart_restore'], 'sep': ['sleep_enter', 'sleep_leave_for_commit', 'sleep_hold_rx', 'sleep_release_rx', 'sleep_exit', 'sleep_kick'], } - texts = dict(shim=shim, mailbox=mailbox, sleep=sleep, pm=pm, sep=sep) + texts = dict(shim=shim, mailbox=mailbox, sleep=sleep, registry=registry, pm=pm, sep=sep) bodies = {n: function(texts[k], n) for k, group in names.items() for n in group} if mutation: name, before, after = MUTATIONS[mutation] @@ -683,11 +762,12 @@ def sources(source, contracts, mutation): c = C_PREFIX + '\n' + '\n'.join(macro(iopoll, n) for n in macros) c += '\n' + '\n'.join(re.search(r'^#define ' + n + r'\s+.*$', shim, re.M).group() for n in ['SEP_PM_POLL_US', 'SEP_PM_TIMEOUT_US']) + c += '\n' + '\n'.join(events) start = shim.index('static struct sep_pm_commit {') c += '\n' + shim[start:shim.index('} sep_pm_commit;', start) + len('} sep_pm_commit;')] # Use the actual provider helper, with only removal/capability dependency state # injected by send for the post-send negative-poll scenarios. - c += '\n' + '\n'.join(bodies[n] for group in ['mailbox', 'sleep', 'shim'] for n in names[group]) + c += '\n' + '\n'.join(bodies[n] for group in ['mailbox', 'sleep', 'registry', 'shim'] for n in names[group]) c += '\n' + C_CONTROLS gate_start = pm.index('#[derive(Clone, Copy, Debug, PartialEq, Eq)]') gate_end = pm.index('/// Stages `request`', gate_start) diff --git a/tools/asahi/t6030-pmp-graph-tests.py b/tools/asahi/t6030-pmp-graph-tests.py index 4c85001acf1a8..2b44c999ab689 100644 --- a/tools/asahi/t6030-pmp-graph-tests.py +++ b/tools/asahi/t6030-pmp-graph-tests.py @@ -49,6 +49,8 @@ def main(): 'static int apple_pmp_report_wait_t6030_14(', 'int apple_pmp_report_wait_supplier_ready('] if 'static bool apple_pmp_report_running(' in source: names.insert(0, 'static bool apple_pmp_report_running(') + if 'static bool apple_pmp_report_hold_locked(' in source: + names.insert(names.index('static int apple_pmp_report_set_state('), 'static bool apple_pmp_report_hold_locked(') if 'static int apple_pmp_report_wait_t6030_bound(' in source: names.insert(names.index('static struct device_node *apple_pmp_report_t6030_entry('), 'static int apple_pmp_report_wait_t6030_bound(') # System sleep (apple,release-for-sleep), where the source has it.