diff --git a/arch/arm64/boot/dts/apple/t6031-dcp.dtsi b/arch/arm64/boot/dts/apple/t6031-dcp.dtsi new file mode 100644 index 0000000000000..518dff478d8e5 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031-dcp.dtsi @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * T6031 internal display. + * + * The same description as t6030-dcp.dtsi: the DCP, its mailbox, the two + * display DARTs, the display subsystem and its PIODMA stream. The T6031 + * display block is the T6030 block at 0x3_8400_0000 instead of + * 0x2_8c00_0000, with every offset inside it unchanged. Addresses, + * interrupts and DART windows are those of the J516C ADT. + * + * Every node stays status = "disabled". The display gate enables the DCP, + * its mailbox, the two DARTs and the display subsystem only when + * apple_t6031_display.enable=1 is on the kernel command line and the boot + * loader has set apple,t6031-handoff = <1>. Compatibles are t6031-specific, + * so a kernel that does not know them leaves the nodes unbound. + * + * Only the T6031 board files include this file: t6034.dtsi includes + * t6031.dtsi, and none of this is known to hold on T6034. + * The panel child is board-specific and is added by the board file. + */ + +/ { + aliases { + /* The boot loader finds the display nodes through these. */ + dcp = &dcp; + disp0 = &display; + disp0-piodma = &disp0_piodma; + disp0_piodma = &disp0_piodma; + }; + + /* The ADT gives no rate for it: this is the T6030 rate, unconfirmed here. */ + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <712000000>; + clock-output-names = "clk_disp0"; + }; +}; + +&soc { + dcp_mbox: mailbox@386c08000 { + compatible = "apple,t6031-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x86c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_disp_fe>; + #mbox-cells = <0>; + status = "disabled"; + }; + + dcp_dart: iommu@38530c000 { + compatible = "apple,t6031-dart", "apple,t8110-dart"; + reg = <0x3 0x8530c000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + disp0_dart: iommu@385304000 { + compatible = "apple,t6031-dart", "apple,t8110-dart"; + reg = <0x3 0x85304000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + /* + * The DCP's ADT clock gate depends on both DISP_CPU and DISP_FE, and + * on T6031 DISP_FE is the child of DISP_CPU: DISP_FE keeps all three + * display power states on. + */ + dcp: dcp@386c00000 { + compatible = "apple,t6031-dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 5>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3", + "disp-4", "disp-5", "iop-vbar"; + reg = <0x3 0x86c00000 0x0 0x4000>, + <0x3 0x84000000 0x0 0x690000>, + <0x3 0x84800000 0x0 0x690000>, + <0x3 0x85320000 0x0 0x4000>, + <0x3 0x85344000 0x0 0x4000>, + <0x3 0x86800000 0x0 0x800000>, + <0x2 0x903d0000 0x0 0x4000>, + <0x3 0x86850000 0x0 0x4000>; + power-domains = <&ps_disp_fe>; + clocks = <&clk_disp0>; + #address-cells = <2>; + #size-cells = <2>; + status = "disabled"; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + }; + }; + + display: display-subsystem { + compatible = "apple,t6031-display-subsystem"; + iommus = <&disp0_dart 0>; + status = "disabled"; + }; +}; + +/* + * The display gate adds the PMP only on display power states the boot + * loader leaves on; the power state driver does not read this. + */ +&ps_disp_sys { + /* The bootloader leaves the internal display powered. */ + apple,inherited-on; +}; + +&ps_disp_fe { + /* The bootloader leaves the internal display powered. */ + apple,inherited-on; +}; + +&ps_disp_cpu { + /* The bootloader leaves the internal display powered. */ + apple,inherited-on; +}; diff --git a/arch/arm64/boot/dts/apple/t6031-gpu.dtsi b/arch/arm64/boot/dts/apple/t6031-gpu.dtsi new file mode 100644 index 0000000000000..2f6a29ca5aabb --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031-gpu.dtsi @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T6031 (M3 Max / G15C) GPU description. + * + * The register ranges and interrupts come from the J516C iBoot ADT: /arm-io + * maps child 0 to 0x2_0000_0000 here, so sgx is at 0x4_0800_0000 and gfx-asc + * at sgx + 0x240_0000, the T6030 offset. The GPU and its mailbox stay + * disabled. + * + * The rest of the description is each machine's, and the boot loader fills it + * from its ADT when it hands the GPU over, as on T8122: the operating points + * (one opp-microvolt cell per cluster, four clusters, and a power each), the + * CS and AFR operating points, the leakage and power-management coefficients, + * the firmware version, the UAT regions below, and the firmware segments, + * which it adds itself as the "fw-text" and "fw-data" memory regions. It + * enables the mailbox and then the GPU node last. Nothing here has a T6030 + * value copied into it. + */ + +/ { + aliases { + gpu = &gpu; /* the boot loader finds the GPU node through this alias */ + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * Placeholder only. The boot loader deletes these children and + * creates one OPP per performance state, with one opp-microvolt + * cell per cluster (the four per-cluster voltage tables of the + * ADT) and opp-microwatt. Until then every value is zero. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0 0 0 0>; + opp-microwatt = <0>; + }; + }; + + /* + * CS and AFR clock domains (ADT cs-perf-states, afr-perf-states). + * Placeholders with one opp-microvolt cell (one die); the boot loader + * replaces the children. + */ + gpu_cs_opp: opp-table-gpu-cs { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + }; + + gpu_afr_opp: opp-table-gpu-afr { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <0>; + }; + }; + + reserved-memory { + /* + * Filled and enabled by the boot loader from the ADT sgx + * gpu-region-*, gfx-shared-region-*, gfx-handoff-* and + * gfx-shared-l2-region-* properties. Disabled until then. + */ + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + status = "disabled"; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + status = "disabled"; + }; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + status = "disabled"; + }; + + uat_pagetables_l2: uat-pagetables-l2 { + reg = <0 0 0 0>; + status = "disabled"; + }; + }; +}; + +&soc { + gpu: gpu@40a400000 { + compatible = "apple,agx-t6031"; + reg = <0x4 0x0a400000 0x0 0x6c000>, + <0x4 0x08000000 0x0 0x3fdc000>; + reg-names = "asc", "sgx"; + interrupt-parent = <&aic>; + /* Preserve the ADT order; entry 5 (IRQ 1249) is marked invalid there. */ + interrupts = , + , + , + , + , + , + ; + /* "gfx" (pmgr_gfx + 0x110), whose parent is afr. */ + power-domains = <&ps_gfx>; + /* + * Do not hold back the ps_gfx sync_state() on this device when it + * never probes, as on the T6030 and T8122. + */ + post-init-providers = <&ps_gfx>; + mboxes = <&agx_mbox>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, + <&uat_handoff>, <&uat_pagetables_l2>; + memory-region-names = "ttbs", "pagetables", "handoff", + "pagetables-l2"; + + /* Zero until the boot loader identifies the loaded GPU firmware. */ + apple,firmware-abi = <0 0 0>; + apple,firmware-version = <0 0 0>; + apple,firmware-compat = <0 0 0>; + + operating-points-v2 = <&gpu_opp>; + apple,cs-opp = <&gpu_cs_opp>; + apple,afr-opp = <&gpu_afr_opp>; + + /* The boot loader sets "okay" as its last step. */ + status = "disabled"; + }; + + /* + * The ASC mailbox at asc + 0x8000. The ADT lists the gfx-asc interrupts + * as 1244, 1243, 1246, 1245; in mailbox order they are 1243..1246, the + * same swap as on T8122. Like the T6030 and T8122 ones it carries no + * asc-mailbox-v4 fallback, so a kernel without M3 Max GPU support + * leaves it unbound even after the boot loader has enabled it. + */ + agx_mbox: mbox@40a408000 { + compatible = "apple,t6031-agx-asc-mailbox"; + reg = <0x4 0x0a408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6031-j514c.dts b/arch/arm64/boot/dts/apple/t6031-j514c.dts index ec53b88aebc98..f4f282e9d3207 100644 --- a/arch/arm64/boot/dts/apple/t6031-j514c.dts +++ b/arch/arm64/boot/dts/apple/t6031-j514c.dts @@ -10,6 +10,7 @@ /dts-v1/; #include "t6031.dtsi" +#include "t6031-dcp.dtsi" #include "t603x-j514-j516.dtsi" / { @@ -29,6 +30,20 @@ brcm,board-type = "apple,texa"; }; +/* + * The J514S values (t6030-j514s.dts): the same chassis and panel. + * Not yet read from a J514C. + */ +&dcp { + panel { + compatible = "apple,panel-j514", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; + apple,max-brightness = <500>; + }; +}; + &aop_audio { apple,chassis-name = "J514"; apple,machine-kind = "MacBook Pro"; diff --git a/arch/arm64/boot/dts/apple/t6031-j516c.dts b/arch/arm64/boot/dts/apple/t6031-j516c.dts index 9e447741cba71..562491755117f 100644 --- a/arch/arm64/boot/dts/apple/t6031-j516c.dts +++ b/arch/arm64/boot/dts/apple/t6031-j516c.dts @@ -10,6 +10,7 @@ /dts-v1/; #include "t6031.dtsi" +#include "t6031-dcp.dtsi" #include "t603x-j514-j516.dtsi" / { @@ -29,6 +30,20 @@ brcm,board-type = "apple,jura"; }; +/* + * The J516S values (t6030-j516s.dts): the same chassis and panel. + * Not yet read from a J516C. + */ +&dcp { + panel { + compatible = "apple,panel-j516", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; + apple,max-brightness = <500>; + }; +}; + &aop_audio { apple,chassis-name = "J516"; apple,machine-kind = "MacBook Pro"; diff --git a/arch/arm64/boot/dts/apple/t6031.dtsi b/arch/arm64/boot/dts/apple/t6031.dtsi index 60df7a3e5eec7..13daaaf994ab6 100644 --- a/arch/arm64/boot/dts/apple/t6031.dtsi +++ b/arch/arm64/boot/dts/apple/t6031.dtsi @@ -47,6 +47,7 @@ #include "t6031-gpio-pins.dtsi" #include "t6031-pmgr.dtsi" +#include "t6031-gpu.dtsi" #undef DIE #undef DIE_NO diff --git a/arch/arm64/boot/dts/apple/t6034.dtsi b/arch/arm64/boot/dts/apple/t6034.dtsi index e47bb7cbad508..4b1dad5263938 100644 --- a/arch/arm64/boot/dts/apple/t6034.dtsi +++ b/arch/arm64/boot/dts/apple/t6034.dtsi @@ -11,6 +11,15 @@ compatible = "apple,t6034", "apple,arm-platform"; }; +/* + * The 14-core M3 Max has its own GPU (30 cores), whose clusters and core masks + * the T6031 description does not give. Its own compatible keeps the T6031 GPU + * support from binding to it even if a boot loader enabled the node. + */ +&gpu { + compatible = "apple,agx-t6034"; +}; + // Memory channels not present on the 14-core variant &ps_amcc6 { status = "disabled"; diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 1726a9448ea3c..63b1d0f298e5c 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -42,6 +42,22 @@ config DRM_APPLE_DCP_T6030 after the display, so userspace should load appledrm explicitly and blacklist its alias autoload. +config DRM_APPLE_DCP_T6031 + bool "MacBook Pro M3 Max (J516C) internal panel" + depends on DRM_APPLE_DCP_T6030 + depends on OF_DYNAMIC + help + Lets the common Apple display driver adopt the J516C (T6031) + internal display coprocessor, which the boot loader hands + over running DCP firmware 14.7.0, with the J514S session and + presentation paths at the T6031 addresses. + + The boot loader describes the display with every node disabled. + The T6031 display handoff enables them, and adds this Mac's PMP + from the values the boot loader passes, only when + apple_t6031_display.enable=1 is on the kernel command line; the + boot framebuffer stays otherwise. + config DRM_APPLE_DCP_T6030_EXT bool "MacBook Pro M3 Pro (J514S) external displays" depends on DRM_APPLE_DCP_T6030 diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 52b4de01d3532..9f8531f32f41b 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -6,7 +6,7 @@ appledrm-y := apple_drv.o apple_resume.o appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o -appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o +appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o dcp-panel-edid.o appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o @@ -24,6 +24,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_dropped_swap_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-panel-edid-test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 2ba772ab0a944..e1662d5755ea8 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -708,6 +708,9 @@ static const struct of_device_id apple_dcp_id_tbl[] = { #endif #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_EXT) { .compatible = "apple,t6030-dcpext" }, +#endif +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6031) + { .compatible = "apple,t6031-dcp" }, #endif {}, }; @@ -997,6 +1000,9 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,display-subsystem" }, #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030) { .compatible = "apple,t6030-display-diagnostics" }, +#endif +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6031) + { .compatible = "apple,t6031-display-subsystem" }, #endif {} }; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 3145e2f5c9489..4e31e979dfbce 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -23,6 +23,7 @@ #include "dcp-announce.h" #include "dcp-display-metadata.h" #include "dcp-metadata-epoch.h" +#include "dcp-panel-edid.h" #include "dcp-180hz.h" #include "iomfb.h" #include "iomfb_h17p.h" @@ -226,6 +227,8 @@ struct dcp_panel { int height_mm; /// panel has a mini-LED backlight bool has_mini_led; + /// make and product for the base EDID, from the panel's compatible + struct dcp_panel_id id; }; enum dcp_iomfb_method_profile { diff --git a/drivers/gpu/drm/apple/dcp-panel-edid-test.c b/drivers/gpu/drm/apple/dcp-panel-edid-test.c new file mode 100644 index 0000000000000..2dc65215f9c5d --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-panel-edid-test.c @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +/* + * The built-in panel's base EDID, read back with the DRM EDID helpers: the + * make and product come from the panel's compatible, and the connector keeps + * the panel's size in millimetres after the EDID update. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "dcp-panel-edid.h" + +#define J516_W_MM 346 +#define J516_H_MM 223 + +static void panel_id_j516(struct kunit *test, struct dcp_panel_id *id) +{ + KUNIT_ASSERT_EQ(test, dcp_panel_id_from_compatible(id, "apple,panel-j516"), 0); +} + +static void panel_id_from_compatible(struct kunit *test) +{ + struct dcp_panel_id id; + + panel_id_j516(test, &id); + KUNIT_EXPECT_STREQ(test, id.vendor, "APP"); + KUNIT_EXPECT_STREQ(test, id.name, "J516"); +} + +static void panel_id_needs_a_model_and_a_known_vendor(struct kunit *test) +{ + struct dcp_panel_id id; + + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel-"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "panel-j516"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "acme,panel-j516"), -ENOENT); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel-j516 x"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, + "apple,panel-j5160000000000"), + -EINVAL); +} + +static const struct drm_edid *panel_edid_j516(struct kunit *test, struct edid *block) +{ + struct dcp_panel_id id; + const struct drm_edid *edid; + + panel_id_j516(test, &id); + KUNIT_ASSERT_EQ(test, dcp_panel_edid_block(&id, J516_W_MM, J516_H_MM, block), 0); + edid = drm_edid_alloc(block, sizeof(*block)); + KUNIT_ASSERT_NOT_NULL(test, edid); + return edid; +} + +static void panel_edid_parses_with_make_product_and_size(struct kunit *test) +{ + struct drm_edid_product_id product; + struct edid block; + const struct drm_edid *edid = panel_edid_j516(test, &block); + const struct edid *raw; + char vendor[4], name[14]; + + KUNIT_EXPECT_TRUE(test, drm_edid_valid(edid)); + drm_edid_get_product_id(edid, &product); + drm_edid_decode_mfg_id(be16_to_cpu(product.manufacturer_name), vendor); + KUNIT_EXPECT_STREQ(test, vendor, "APP"); + KUNIT_EXPECT_EQ(test, le16_to_cpu(product.product_code), 0); + KUNIT_EXPECT_EQ(test, le32_to_cpu(product.serial_number), 0); + + raw = drm_edid_raw(edid); + KUNIT_ASSERT_NOT_NULL(test, raw); + drm_edid_get_monitor_name(raw, name, sizeof(name)); + KUNIT_EXPECT_STREQ(test, name, "J516"); + KUNIT_EXPECT_EQ(test, raw->width_cm, 35); + KUNIT_EXPECT_EQ(test, raw->height_cm, 22); + KUNIT_EXPECT_EQ(test, raw->extensions, 0); + drm_edid_free(edid); +} + +static void panel_edid_has_no_serial_and_no_timings(struct kunit *test) +{ + struct edid block; + const struct drm_edid *edid = panel_edid_j516(test, &block); + unsigned int i, names = 0; + + for (i = 0; i < ARRAY_SIZE(block.detailed_timings); i++) { + const struct detailed_timing *d = &block.detailed_timings[i]; + + KUNIT_EXPECT_EQ(test, le16_to_cpu(d->pixel_clock), 0); + KUNIT_EXPECT_NE(test, d->data.other_data.type, EDID_DETAIL_MONITOR_SERIAL); + KUNIT_EXPECT_NE(test, d->data.other_data.type, EDID_DETAIL_MONITOR_RANGE); + names += d->data.other_data.type == EDID_DETAIL_MONITOR_NAME; + } + KUNIT_EXPECT_EQ(test, names, 1); + for (i = 0; i < sizeof(block.established_timings); i++) + KUNIT_EXPECT_EQ(test, ((u8 *)&block.established_timings)[i], 0); + drm_edid_free(edid); +} + +static void panel_edid_refuses_a_size_it_cannot_hold(struct kunit *test) +{ + struct dcp_panel_id id; + struct edid block; + + panel_id_j516(test, &id); + KUNIT_EXPECT_EQ(test, dcp_panel_edid_block(&id, 0, J516_H_MM, &block), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_edid_block(&id, J516_W_MM, 2560, &block), -EINVAL); + id.vendor[0] = '\0'; + KUNIT_EXPECT_EQ(test, dcp_panel_edid_block(&id, J516_W_MM, J516_H_MM, &block), + -EINVAL); +} + +struct panel_edid_test { + struct drm_device drm; + struct drm_connector connector; +}; + +static const struct drm_connector_funcs panel_edid_connector_funcs = {}; + +static void panel_edid_update_keeps_the_size_in_mm(struct kunit *test) +{ + struct panel_edid_test *t; + struct drm_connector *connector; + struct dcp_panel_id id; + struct device *dev; + int ret; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + t = drm_kunit_helper_alloc_drm_device(test, dev, struct panel_edid_test, drm, + DRIVER_MODESET); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t); + connector = &t->connector; + ret = drmm_connector_init(&t->drm, connector, &panel_edid_connector_funcs, + DRM_MODE_CONNECTOR_eDP, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + + panel_id_j516(test, &id); + mutex_lock(&t->drm.mode_config.mutex); + ret = dcp_panel_edid_update(connector, &id, J516_W_MM, J516_H_MM); + mutex_unlock(&t->drm.mode_config.mutex); + KUNIT_ASSERT_EQ(test, ret, 0); + + /* The EDID update sets 350x220 from the block; the panel's own size wins. */ + KUNIT_EXPECT_EQ(test, connector->display_info.width_mm, J516_W_MM); + KUNIT_EXPECT_EQ(test, connector->display_info.height_mm, J516_H_MM); + KUNIT_ASSERT_NOT_NULL(test, connector->edid_blob_ptr); + KUNIT_EXPECT_EQ(test, connector->edid_blob_ptr->length, EDID_LENGTH); +} + +static struct kunit_case panel_edid_cases[] = { + KUNIT_CASE(panel_id_from_compatible), + KUNIT_CASE(panel_id_needs_a_model_and_a_known_vendor), + KUNIT_CASE(panel_edid_parses_with_make_product_and_size), + KUNIT_CASE(panel_edid_has_no_serial_and_no_timings), + KUNIT_CASE(panel_edid_refuses_a_size_it_cannot_hold), + KUNIT_CASE(panel_edid_update_keeps_the_size_in_mm), + {} +}; + +static struct kunit_suite panel_edid_suite = { + .name = "apple-dcp-panel-edid", + .test_cases = panel_edid_cases, +}; +kunit_test_suite(panel_edid_suite); diff --git a/drivers/gpu/drm/apple/dcp-panel-edid.c b/drivers/gpu/drm/apple/dcp-panel-edid.c new file mode 100644 index 0000000000000..9d816697184e0 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-panel-edid.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * A base EDID for a built-in panel. + * + * The DCP gives no EDID for a built-in panel, so a compositor sees no make, + * model or physical size for it. This builds one 128-byte block from what the + * panel's device-tree node says: the vendor and model of its most specific + * compatible ("apple,panel-j516": make APP, name J516) and its size. + * + * The block makes no other claim: no product code or serial number, no + * manufacture date, no colour primaries, no timings and no refresh range. The + * modes and the refresh range stay the firmware's. An EDID holds the size in + * whole centimetres, so the connector gets back the size in millimetres + * after the update. + */ + +#include +#include +#include +#include + +#include +#include + +#include "dcp-panel-edid.h" + +/* Device-tree vendor prefixes and their PNP IDs. */ +static const struct { + const char *prefix; + const char *pnp; +} dcp_panel_vendors[] = { + { "apple", "APP" }, +}; + +int dcp_panel_id_from_compatible(struct dcp_panel_id *id, const char *compatible) +{ + const char *comma = strchr(compatible, ','), *model; + size_t vendor_len, len, i; + + memset(id, 0, sizeof(*id)); + if (!comma) + return -EINVAL; + vendor_len = comma - compatible; + model = comma + 1; + if (str_has_prefix(model, "panel-")) + model += strlen("panel-"); + else + return -EINVAL; + len = strlen(model); + if (!len || len > DCP_PANEL_NAME_LEN) + return -EINVAL; + for (i = 0; i < len; i++) { + if (!isalnum(model[i]) && model[i] != '-') + return -EINVAL; + id->name[i] = toupper(model[i]); + } + + for (i = 0; i < ARRAY_SIZE(dcp_panel_vendors); i++) { + if (strlen(dcp_panel_vendors[i].prefix) == vendor_len && + !strncmp(compatible, dcp_panel_vendors[i].prefix, vendor_len)) { + strscpy(id->vendor, dcp_panel_vendors[i].pnp); + return 0; + } + } + memset(id, 0, sizeof(*id)); + return -ENOENT; +} + +int dcp_panel_edid_block(const struct dcp_panel_id *id, int width_mm, int height_mm, + struct edid *block) +{ + struct detailed_non_pixel *name; + u16 manufacturer = 0; + size_t len = strnlen(id->name, sizeof(id->name)); + u8 sum = 0; + unsigned int i; + + if (width_mm <= 0 || height_mm <= 0 || + DIV_ROUND_CLOSEST(width_mm, 10) > 255 || DIV_ROUND_CLOSEST(height_mm, 10) > 255) + return -EINVAL; + if (strnlen(id->vendor, sizeof(id->vendor)) != 3 || !len || len > DCP_PANEL_NAME_LEN) + return -EINVAL; + for (i = 0; i < 3; i++) { + if (id->vendor[i] < 'A' || id->vendor[i] > 'Z') + return -EINVAL; + manufacturer = manufacturer << 5 | (id->vendor[i] - '@'); + } + + memset(block, 0, sizeof(*block)); + memcpy(block->header, "\x00\xff\xff\xff\xff\xff\xff\x00", sizeof(block->header)); + block->product_id.manufacturer_name = cpu_to_be16(manufacturer); + block->version = 1; + block->revision = 4; + block->input = DRM_EDID_INPUT_DIGITAL; + block->width_cm = DIV_ROUND_CLOSEST(width_mm, 10); + block->height_cm = DIV_ROUND_CLOSEST(height_mm, 10); + /* Standard timings 0x0101 are unused slots. */ + memset(block->standard_timings, 0x01, sizeof(block->standard_timings)); + + /* The first descriptor names the panel; the other three are dummies. */ + name = &block->detailed_timings[0].data.other_data; + name->type = EDID_DETAIL_MONITOR_NAME; + memset(name->data.str.str, ' ', sizeof(name->data.str.str)); + memcpy(name->data.str.str, id->name, len); + if (len < sizeof(name->data.str.str)) + name->data.str.str[len] = '\n'; + for (i = 1; i < ARRAY_SIZE(block->detailed_timings); i++) + block->detailed_timings[i].data.other_data.type = 0x10; + + for (i = 0; i < sizeof(*block) - 1; i++) + sum += ((const u8 *)block)[i]; + block->checksum = -sum; + return 0; +} + +int dcp_panel_edid_update(struct drm_connector *connector, const struct dcp_panel_id *id, + int width_mm, int height_mm) +{ + const struct drm_edid *edid; + struct edid block; + int ret; + + ret = dcp_panel_edid_block(id, width_mm, height_mm, &block); + if (ret) + return ret; + edid = drm_edid_alloc(&block, sizeof(block)); + if (!edid) + return -ENOMEM; + ret = drm_edid_connector_update(connector, edid); + drm_edid_free(edid); + /* The update set the size from the block, in whole centimetres. */ + connector->display_info.width_mm = width_mm; + connector->display_info.height_mm = height_mm; + return ret; +} diff --git a/drivers/gpu/drm/apple/dcp-panel-edid.h b/drivers/gpu/drm/apple/dcp-panel-edid.h new file mode 100644 index 0000000000000..58c865039ad3a --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-panel-edid.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* A base EDID for a built-in panel, from its device-tree node. */ +#ifndef __APPLE_DCP_PANEL_EDID_H__ +#define __APPLE_DCP_PANEL_EDID_H__ + +#include + +/* The monitor-name text of an EDID descriptor holds 13 characters. */ +#define DCP_PANEL_NAME_LEN 13 + +/* The make and product of a panel, as its compatible gives them. */ +struct dcp_panel_id { + char vendor[4]; /* PNP ID, empty if unknown */ + char name[DCP_PANEL_NAME_LEN + 1]; /* monitor name */ +}; + +struct drm_connector; +struct edid; + +int dcp_panel_id_from_compatible(struct dcp_panel_id *id, const char *compatible); +int dcp_panel_edid_block(const struct dcp_panel_id *id, int width_mm, int height_mm, + struct edid *block); +int dcp_panel_edid_update(struct drm_connector *connector, const struct dcp_panel_id *id, + int width_mm, int height_mm); + +#endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ef60731b30ec0..0148cb564bb13 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2458,7 +2458,9 @@ static bool dcp_v14_7_reserved(struct apple_dcp *dcp, u64 pa, size_t size, for (i = 0; (node = of_parse_phandle(dcp->dev->of_node, "memory-region", i)); i++) { struct reserved_mem *rmem = of_reserved_mem_lookup(node); - if (of_device_is_compatible(node, "apple,asc-mem") && + /* The T6031 handoff marks the OS log carveout without the compatible. */ + if ((of_device_is_compatible(node, "apple,asc-mem") || + (oslog && of_property_read_bool(node, "apple,dcp-os-log"))) && of_property_read_bool(node, "no-map") && oslog == of_property_read_bool(node, "apple,dcp-os-log") && !of_address_to_resource(node, 0, &res) && rmem && @@ -3592,19 +3594,43 @@ static int dcp_enable_dp2hdmi_hpd(struct apple_dcp *dcp) #define DCP_V14_7_START_TIMEOUT msecs_to_jiffies(10000) #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030) +/* + * On a board whose PMP the display handoff may leave out (T6031 with + * apple_t6031_display.pmp=0) the DCP is adopted as the boot loader left it, + * with no PMP acknowledgement to wait for. + */ +static bool dcp_v14_7_without_pmp(struct apple_dcp *dcp) +{ + return dcp->v14_7 && dcp->v14_7->pmp_optional && !apple_pmp_display_present(); +} + static int dcp_t6030_display_prepare(struct device *dev) { - return apple_pmp_j514s_display_prepare(dev); + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp_v14_7_without_pmp(dcp)) { + dev_warn(dev, "no PMP described: adopting the DCP without a PMP display hold\n"); + return 0; + } + return apple_pmp_display_prepare(dev); } static int dcp_t6030_display_get(struct device *dev) { - return apple_pmp_j514s_display_get(dev); + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp_v14_7_without_pmp(dcp)) + return 0; + return apple_pmp_display_get(dev); } static int dcp_t6030_display_check(struct device *dev) { - return apple_pmp_j514s_display_check(dev); + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp_v14_7_without_pmp(dcp)) + return 0; + return apple_pmp_display_check(dev); } #else static int dcp_t6030_display_prepare(struct device *dev) { return -ENODEV; } @@ -4538,6 +4564,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) if (panel_np) { const char height_prop[2][16] = { "adj-height-mm", "height-mm" }; + const char *compatible; if (of_device_is_available(panel_np)) { ret = of_property_read_u32(panel_np, "apple,max-brightness", @@ -4550,6 +4577,10 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) /* use adjusted height as long as the notch is hidden */ of_property_read_u32(panel_np, height_prop[!dcp->notch_height], &dcp->panel.height_mm); + /* The most specific compatible names the panel for its EDID. */ + if (!of_property_read_string_index(panel_np, "compatible", 0, &compatible) && + dcp_panel_id_from_compatible(&dcp->panel.id, compatible)) + dev_dbg(dev, "no EDID make and product in '%s'\n", compatible); of_node_put(panel_np); dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; @@ -4808,18 +4839,105 @@ static const struct component_ops dcp_comp_ops = { .unbind = dcp_comp_unbind, }; -/* The J514S (T6030) handoff this driver adopts. */ -#define J514S_DCP_UUID "DDF38191-93B3-324A-BC8F-643006F5AC82" -#define J514S_DCP_COPROC 0x28ec00000ULL -#define J514S_NOTCH_HEIGHT 74 -#define J514S_BOOT_STRIDE 12096 -#define J514S_INHERITED "apple,j514s-inherited-mappings" +/* + * The 14.7 panel handoffs this driver adopts: the J514S (T6030) and the + * J516C (T6031). Each names its board, the DCP image the boot loader + * leaves running, the coprocessor window, the panel and the boot framebuffer + * the boot loader draws on it, and the form of its handoff: + * - the J514S boot loader enables the display nodes itself and marks the + * inherited mappings on the display subsystem and the PIODMA child; + * - the T6031 boot loader leaves every node disabled and marks the DCP, the + * display subsystem and the PIODMA child with apple,t6031-handoff = <1>; + * the T6031 display handoff (drivers/soc/apple) enables them on request. + */ +struct dcp_v14_7_board { + const char *name; + const char *soc; /* root compatible */ + const char *const *boards; /* root compatibles, NULL-terminated */ + const char *dcp_compat; + const char *display_compat; + const char *uuid; /* the DCP image (apple,firmware-uuid) */ + u64 coproc; + u32 panel_width, panel_height, notch_height; + u32 boot_stride; + const char *marker; /* u32 1 on dcp, display and piodma */ + const char *inherited; /* boolean on display and piodma */ + const char *native_scanout, *native_piodma, *chosen_note; + bool pmp_optional; /* adopt without a PMP when none is described */ + bool swap_timestamps; /* panel swaps carry constant timestamps */ +}; + +static const char *const j514s_boards[] = { "apple,j514s", NULL }; +static const char *const t6031_boards[] = { "apple,j516c", NULL }; + +static const struct dcp_v14_7_board dcp_v14_7_j514s = { + .name = "J514S", + .soc = "apple,t6030", + .boards = j514s_boards, + .dcp_compat = "apple,t6030-dcp", + .display_compat = "apple,t6030-display-diagnostics", + .uuid = "DDF38191-93B3-324A-BC8F-643006F5AC82", + .coproc = 0x28ec00000ULL, + .panel_width = 3024, + .panel_height = 1964, + .notch_height = 74, + .boot_stride = 12096, + .inherited = "apple,j514s-inherited-mappings", + .native_scanout = "apple,j514s-native-scanout", + .native_piodma = "apple,j514s-native-piodma", + .chosen_note = "aurora,j514s-display-handoff", +}; + +/* + * MacBook Pro 16" M3 Max (J516C): the J516 panel, 3456x2234 with 74 notch + * rows above the 3456x2160 boot framebuffer at 4 bytes per pixel. The image + * is the one this Mac's ADT and the boot loader's apple,firmware-uuid name. + * The 14" M3 Max (J514C) has another panel and needs its own profile. + */ +static const struct dcp_v14_7_board dcp_v14_7_t6031 = { + .name = "J516C", + .soc = "apple,t6031", + .boards = t6031_boards, + .dcp_compat = "apple,t6031-dcp", + .display_compat = "apple,t6031-display-subsystem", + .uuid = "F493F03E-CC40-3D89-9A08-5F259BA2B246", + .coproc = 0x386c00000ULL, + .panel_width = 3456, + .panel_height = 2234, + .notch_height = 74, + .boot_stride = 3456 * 4, + .marker = "apple,t6031-handoff", + .pmp_optional = true, + .swap_timestamps = true, +}; + +/* The board profile of a 14.7 panel DCP node, or NULL. */ +static const struct dcp_v14_7_board *dcp_v14_7_board(struct device_node *np) +{ + static const struct dcp_v14_7_board *const boards[] = { + &dcp_v14_7_j514s, +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6031) + &dcp_v14_7_t6031, +#endif + }; + int i, j; + + for (i = 0; i < ARRAY_SIZE(boards); i++) { + if (!of_device_is_compatible(np, boards[i]->dcp_compat) || + !of_machine_is_compatible(boards[i]->soc)) + continue; + for (j = 0; boards[i]->boards[j]; j++) + if (of_machine_is_compatible(boards[i]->boards[j])) + return boards[i]; + } + return NULL; +} /* * The boot loader leaves one simple framebuffer on the panel: the full panel * or the panel below the notch, at the stride D101 reports. */ -static int dcp_t6030_boot_framebuffer(struct device *dev, u32 *stride) +static int dcp_v14_7_boot_framebuffer(struct device *dev, const struct dcp_v14_7_board *b) { struct device_node *np; u32 width, height, pitch; @@ -4832,9 +4950,9 @@ static int dcp_t6030_boot_framebuffer(struct device *dev, u32 *stride) of_property_read_u32(np, "width", &width) || of_property_read_u32(np, "height", &height) || of_property_read_u32(np, "stride", &pitch) || - width != J514S_PANEL_WIDTH || pitch != J514S_BOOT_STRIDE || - (height != J514S_PANEL_HEIGHT && - height != J514S_PANEL_HEIGHT - J514S_NOTCH_HEIGHT)) { + width != b->panel_width || pitch != b->boot_stride || + (height != b->panel_height && + height != b->panel_height - b->notch_height)) { of_node_put(np); dev_err(dev, "unexpected boot framebuffer %pOF\n", np); return -EINVAL; @@ -4844,17 +4962,31 @@ static int dcp_t6030_boot_framebuffer(struct device *dev, u32 *stride) dev_err(dev, "no boot framebuffer\n"); return -ENODEV; } - *stride = J514S_BOOT_STRIDE; return 0; } /* An enabled handoff node at @path that carries the inherited-mapping marker. */ static bool dcp_t6030_handoff_node(struct device_node *np, const char *path, - const char *compatible) + const char *compatible, const char *inherited) { struct device_node *alias = of_find_node_by_path(path); bool ok = np && alias == np && of_device_is_available(np) && - of_property_read_bool(np, J514S_INHERITED) && + of_property_read_bool(np, inherited) && + (!compatible || of_device_is_compatible(np, compatible)); + + of_node_put(alias); + return ok; +} + +/* An enabled handoff node at @path marked with the SoC's handoff word. */ +static bool dcp_t6031_handoff_node(struct device_node *np, const char *path, + const char *compatible, const char *marker) +{ + struct device_node *alias = of_find_node_by_path(path); + u32 val; + bool ok = np && alias == np && of_device_is_available(np) && + !of_property_read_u32(np, marker, &val) && val == 1 && + of_property_present(np, "memory-region") && (!compatible || of_device_is_compatible(np, compatible)); of_node_put(alias); @@ -4862,37 +4994,38 @@ static bool dcp_t6030_handoff_node(struct device_node *np, const char *path, } /* - * J514S admission: board and firmware identity, the DCP's coprocessor + * 14.7 panel admission: board and firmware identity, the DCP's coprocessor * window and DMA domain, the boot loader's display handoff (scanout and - * PIODMA markers, inherited mappings, the chosen handoff note) and the boot - * framebuffer. The DCP is only adopted from this exact handoff. + * PIODMA, in the board's form) and the boot framebuffer. The DCP is only + * adopted from this exact handoff. */ -static int dcp_t6030_admit(struct platform_device *pdev, u32 *stride) +static int dcp_v14_7_admit(struct platform_device *pdev, const struct dcp_v14_7_board **board) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; struct device_node *piodma, *disp, *chosen; + const struct dcp_v14_7_board *b; struct resource *res; const char *uuid; u32 compat[3]; bool handoff; int ret; - if (!of_machine_is_compatible("apple,j514s") || - !of_machine_is_compatible("apple,t6030")) { - dev_info(dev, "not a J514S; not binding\n"); + b = dcp_v14_7_board(np); + if (!b) { + dev_info(dev, "not a board with a known 14.7 panel handoff; not binding\n"); return -ENODEV; } if (of_property_count_u32_elems(np, "apple,firmware-compat") != 3 || of_property_read_u32_array(np, "apple,firmware-compat", compat, 3) || compat[0] != 14 || compat[1] != 7 || compat[2] != 0 || of_property_read_string(np, "apple,firmware-uuid", &uuid) || - strcmp(uuid, J514S_DCP_UUID)) { - dev_info(dev, "DCP firmware is not the qualified 14.7.0 image; not binding\n"); + strcmp(uuid, b->uuid)) { + dev_info(dev, "DCP firmware is not the %s 14.7.0 image; not binding\n", b->name); return -ENODEV; } res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "coproc"); - if (!res || res->start != J514S_DCP_COPROC || resource_size(res) != SZ_16K) { + if (!res || res->start != b->coproc || resource_size(res) != SZ_16K) { dev_info(dev, "unexpected coprocessor window; not binding\n"); return -ENODEV; } @@ -4901,15 +5034,23 @@ static int dcp_t6030_admit(struct platform_device *pdev, u32 *stride) piodma = of_get_child_by_name(np, "piodma"); disp = of_find_node_by_path("disp0"); - chosen = of_find_node_by_path("/chosen"); - handoff = of_property_read_bool(np, "apple,j514s-native-scanout") && - of_property_read_bool(np, "apple,j514s-native-piodma") && - dcp_t6030_handoff_node(piodma, "disp0_piodma", NULL) && - dcp_t6030_handoff_node(disp, "disp0", - "apple,t6030-display-diagnostics") && - chosen && - of_property_present(chosen, "aurora,j514s-display-handoff"); - of_node_put(chosen); + if (b->marker) { + u32 val; + + handoff = !of_property_read_u32(np, b->marker, &val) && val == 1 && + of_property_present(np, "memory-region") && + dcp_t6031_handoff_node(piodma, "disp0_piodma", NULL, b->marker) && + dcp_t6031_handoff_node(disp, "disp0", b->display_compat, b->marker); + } else { + chosen = of_find_node_by_path("/chosen"); + handoff = of_property_read_bool(np, b->native_scanout) && + of_property_read_bool(np, b->native_piodma) && + dcp_t6030_handoff_node(piodma, "disp0_piodma", NULL, b->inherited) && + dcp_t6030_handoff_node(disp, "disp0", b->display_compat, + b->inherited) && + chosen && of_property_present(chosen, b->chosen_note); + of_node_put(chosen); + } of_node_put(disp); of_node_put(piodma); if (!handoff) { @@ -4917,11 +5058,12 @@ static int dcp_t6030_admit(struct platform_device *pdev, u32 *stride) return -ENODEV; } - ret = dcp_t6030_boot_framebuffer(dev, stride); + ret = dcp_v14_7_boot_framebuffer(dev, b); if (ret) return ret; - dev_info(dev, "J514S DCP firmware 14.7.0 handoff admitted\n"); + dev_info(dev, "%s DCP firmware 14.7.0 handoff admitted\n", b->name); + *board = b; return 0; } @@ -4931,7 +5073,7 @@ static int dcp_platform_probe(struct platform_device *pdev) enum dcp_firmware_version fw_compat; struct device *dev = &pdev->dev; struct apple_dcp *dcp; - u32 boot_stride = 0; + const struct dcp_v14_7_board *board = NULL; int ret, surf, num_surfs; u32 surf_en; u32 mux_index; @@ -4953,7 +5095,7 @@ static int dcp_platform_probe(struct platform_device *pdev) /* J514S: the panel or a DCPEXT, each only from its qualified handoff. */ if (IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030) && fw_compat == DCP_FIRMWARE_V_14_7) { ret = IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_EXT) && hw->v14_7_external ? - dcp_t6030_ext_admit(pdev) : dcp_t6030_admit(pdev, &boot_stride); + dcp_t6030_ext_admit(pdev) : dcp_v14_7_admit(pdev, &board); if (ret) return ret; } @@ -5013,7 +5155,13 @@ static int dcp_platform_probe(struct platform_device *pdev) ret = iomfb_v14_7_session_init(dcp); if (ret) return ret; - dcp->v14_7->boot_stride = boot_stride; + if (board) { + dcp->v14_7->boot_stride = board->boot_stride; + dcp->v14_7->panel_width = board->panel_width; + dcp->v14_7->panel_height = board->panel_height; + dcp->v14_7->pmp_optional = board->pmp_optional; + dcp->v14_7->swap_timestamps = board->swap_timestamps; + } } /* The domain use must survive driver-core DMA cleanup for a live session. */ @@ -5465,6 +5613,26 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t6030 = { }; #endif +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6031) +/* + * MacBook Pro 16" M3 Max (J516C, T6031) internal panel: the J514S session + * at the T6031 addresses. The panel's size and brightness come from the + * panel child the T6031 device tree carries; the hardware data are the + * same mini-LED facts in case it is missing. + */ +static const struct apple_dcp_hw_data apple_dcp_hw_t6031 = { + .num_dptx_ports = 0, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_V_14_7, + .primary_only = true, + .optional_clock = true, + .no_rt_bandwidth = true, + .panel_mini_led = true, + .panel_max_nits = 500, + .panel_default_nits = 140, +}; +#endif + #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_EXT) /* * MacBook Pro M3 Pro (J514S) external display controllers (DCPEXT0/1): the @@ -5500,6 +5668,9 @@ static const struct of_device_id of_match[] = { #endif #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_EXT) { .compatible = "apple,t6030-dcpext", .data = &apple_dcp_hw_t6030_dcpext, }, +#endif +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6031) + { .compatible = "apple,t6031-dcp", .data = &apple_dcp_hw_t6031, }, #endif { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5e6aed367f3fb..fbcfc0f5c35f7 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1655,6 +1655,13 @@ int dcp_get_modes(struct drm_connector *connector) } else { apple_connector->drm_edid_firmware_derived = false; } + } else if (nr_modes && iomfb_is_v14_7_panel(dcp) && dcp->panel.id.vendor[0]) { + /* No EDID comes for this panel: publish the base block of its node. */ + int update = dcp_panel_edid_update(connector, &dcp->panel.id, + dcp->panel.width_mm, dcp->panel.height_mm); + + if (update) + dev_warn_once(dcp->dev, "panel EDID publication failed: %d\n", update); } else { u64 generation = 0; const struct drm_edid *derived = nr_modes ? diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 8ae2b3dc43b34..befe2e36833cc 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -3175,6 +3175,27 @@ void iomfb_present_backlight_v14_7(struct apple_dcp *dcp) do_swap(dcp, NULL, NULL); } +/* + * Swap timestamps for the built-in panel. iomfb_flush() fills them from the + * counter for its ProMotion panels; on this SoC the counter runs at 1 GHz and + * such values hang the 14.7 firmware's swap. Use the constant 120 in the + * same three slots, a time that is never in the future. Run after the + * request is encoded, since the encoder zeroes the whole request. Only the + * main display of a board profile that asks for them (T6031) gets them; the + * J514S panel, external DCPs and DCPs without a 14.7 board profile keep + * zero timestamps. + */ +static void iomfb_swap_timestamps_v14_7(struct apple_dcp *dcp, + struct dcp_swap_submit_req_v14_7 *req) +{ + if (!dcp->main_display || !dcp->v14_7 || !dcp->v14_7->swap_timestamps) + return; + + req->swap.timestamp[0] = 120; + req->swap.timestamp[1] = 120; + req->swap.timestamp[6] = 120; +} + /* * Complete a request just before A407: the single-surface invariant and the * level the backlight policy has to present. False fails the present @@ -3196,6 +3217,7 @@ bool iomfb_swap_prepare_v14_7(struct apple_dcp *dcp, dcp->iomfb_v14_7.refused++; return false; } + iomfb_swap_timestamps_v14_7(dcp, req); if (!dcp_backlight_active(dcp)) return true; if (!transaction) @@ -3218,6 +3240,7 @@ bool iomfb_swap_prepare_v14_7(struct apple_dcp *dcp, swap_id = req->swap.swap_id; iomfb_v14_7_encode_present(req, NULL); req->swap.swap_id = swap_id; + iomfb_swap_timestamps_v14_7(dcp, req); dcp->iomfb_v14_7.blank = true; } if (iomfb_v14_7_encode_nits(req, true, present.nits)) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index bf421c0052793..c3044675cd159 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -427,9 +427,11 @@ static int v14_7_preferred_timing(struct apple_dcp *dcp, const void *blob, u32 s preferred = &modes[i]; } refresh = preferred ? drm_mode_vrefresh(&preferred->mode) : 0; - if (!preferred || preferred->mode.hdisplay != J514S_PANEL_WIDTH || - preferred->mode.vdisplay != J514S_PANEL_HEIGHT || + if (!preferred || preferred->mode.hdisplay != dcp->v14_7->panel_width || + preferred->mode.vdisplay != dcp->v14_7->panel_height || refresh < 59 || refresh > 121) { + dev_err(dcp->dev, "preferred timing is not the %ux%u panel at 60-120 Hz\n", + dcp->v14_7->panel_width, dcp->v14_7->panel_height); kfree(modes); return -EINVAL; } diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index 311aefd7e1dc8..9a390e086bbce 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -76,9 +76,7 @@ bool iomfb_check_callback_v14_7(int tag, u32 in_len, u32 out_len); struct apple_dcp; struct dcp_property_store; -/* The J514S panel and the DCP's DMA window (DART SID5, 64 GiB at 1 TiB). */ -#define J514S_PANEL_WIDTH 3024 -#define J514S_PANEL_HEIGHT 1964 +/* The DCP's DMA window (DART SID5, 64 GiB at 1 TiB). */ #define DCP_V14_7_DVA_BASE SZ_1T #define DCP_V14_7_DVA_SIZE SZ_64G @@ -108,8 +106,13 @@ struct dcp_v14_7_session { /* RTKit system buffers (borrowed and allocated), bounded like D451. */ size_t rtkit_bytes; - /* Stride of the inherited boot framebuffer, checked at admission. */ + /* The board's panel and boot framebuffer, set at admission. */ u32 boot_stride; + u32 panel_width, panel_height; + /* The DCP may be adopted with no PMP described (T6031 experiment). */ + bool pmp_optional; + /* The panel's swaps carry constant timestamps (T6031 only). */ + bool swap_timestamps; /* IOMFB READY word; startup progress. */ u64 ready_word; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_test.c b/drivers/gpu/drm/apple/iomfb_v14_7_test.c index 9e90aa402cd1c..65db767dd29dc 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_test.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7_test.c @@ -413,6 +413,101 @@ static void v14_7_policy_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, READ_ONCE(dcp->crashed)); } +/* + * Which DCPs stamp their swaps: the main display of a board profile that + * asks for it (T6031). The J514S panel, the external role and a DCP with + * no 14.7 board profile send zero timestamps, byte for byte as before. + */ +struct v14_7_ts_case { + const char *name; + bool profile; /* a 14.7 board profile was admitted */ + bool swap_timestamps; /* the profile asks for timestamps */ + bool main_display; + bool stamped; +}; + +static const struct v14_7_ts_case v14_7_ts_cases[] = { + { "T6031 main display", true, true, true, true }, + { "J514S main display", true, false, true, false }, + { "T6031 profile, external role", true, true, false, false }, + { "no 14.7 board profile", false, false, true, false }, +}; + +static void v14_7_ts_case_desc(const struct v14_7_ts_case *c, char *desc) +{ + strscpy(desc, c->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(v14_7_ts_case, v14_7_ts_cases, v14_7_ts_case_desc); + +static void v14_7_expect_timestamps(struct kunit *test, + const struct dcp_swap_submit_req_v14_7 *req, + bool stamped, const char *present) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(req->swap.timestamp); i++) { + u64 want = stamped && (i == 0 || i == 1 || i == 6) ? 120 : 0; + + KUNIT_EXPECT_EQ_MSG(test, req->swap.timestamp[i], want, + "%s: slot %u", present, i); + } +} + +static void v14_7_swap_timestamps_test(struct kunit *test) +{ + const struct v14_7_ts_case *c = test->param_value; + struct iomfb_transaction atomic = { .presentation = true }; + struct iomfb_transaction brightness = { + .presentation = true, .brightness_only = true, + }; + struct dcp_swap_submit_req_v14_7 *frame; + struct iomfb_scanout_v14_7 *scanout; + struct apple_dcp *dcp; + + if (!IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030)) + kunit_skip(test, "needs DRM_APPLE_DCP_T6030"); + + dcp = v14_7_dcp(test); + dcp->main_display = c->main_display; + if (c->profile) { + dcp->v14_7 = kunit_kzalloc(test, sizeof(*dcp->v14_7), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dcp->v14_7); + dcp->v14_7->swap_timestamps = c->swap_timestamps; + } + frame = kunit_kzalloc(test, SWAP_SIZE, GFP_KERNEL); + scanout = kunit_kzalloc(test, sizeof(*scanout), GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, frame && scanout); + iomfb_v14_7_encode_present(frame, &v14_7_frames[1]); + + /* An atomic present before the backlight is set up. */ + KUNIT_ASSERT_TRUE(test, v14_7_prepare(dcp, &atomic, frame, 0x2a)); + v14_7_expect_timestamps(test, &dcp->swap_v14_7, c->stamped, "atomic"); + if (!c->stamped) + KUNIT_EXPECT_EQ(test, v14_7_crc(&dcp->swap_v14_7), 0xf38285f2U); + + /* The first present with a level. */ + iomfb_configure_backlight_v14_7(dcp); + flush_work(&dcp->bl_register_wq); + KUNIT_ASSERT_TRUE(test, dcp_backlight_active(dcp)); + KUNIT_ASSERT_TRUE(test, v14_7_prepare(dcp, &atomic, frame, 7)); + v14_7_expect_timestamps(test, &dcp->swap_v14_7, c->stamped, "level"); + if (!c->stamped) + KUNIT_EXPECT_EQ(test, v14_7_crc(&dcp->swap_v14_7), 0xd05cf8c9U); + KUNIT_ASSERT_TRUE(test, atomic.backlight_reserved); + KUNIT_ASSERT_TRUE(test, dcp_backlight_complete(dcp, atomic.backlight.sequence, true)); + + /* The soft-DPMS blank, re-encoded from the completed frame. */ + scanout->request = *frame; + dcp->iomfb_v14_7.scanout = scanout; + v14_7_bl(dcp, -1, 0); + KUNIT_ASSERT_TRUE(test, v14_7_prepare(dcp, &brightness, &scanout->request, 0x1001)); + KUNIT_EXPECT_TRUE(test, dcp->iomfb_v14_7.blank); + v14_7_expect_timestamps(test, &dcp->swap_v14_7, c->stamped, "DPMS blank"); + if (!c->stamped) + KUNIT_EXPECT_EQ(test, v14_7_crc(&dcp->swap_v14_7), 0xdb3c1a63U); +} + static struct kunit_case v14_7_present_cases[] = { KUNIT_CASE_PARAM(v14_7_present_encoding_test, v14_7_case_gen_params), KUNIT_CASE(v14_7_layout_test), @@ -420,6 +515,7 @@ static struct kunit_case v14_7_present_cases[] = { KUNIT_CASE(v14_7_frame_check_test), KUNIT_CASE(v14_7_single_surface_test), KUNIT_CASE(v14_7_policy_test), + KUNIT_CASE_PARAM(v14_7_swap_timestamps_test, v14_7_ts_case_gen_params), {} }; diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 968ff7820f38c..c8b9564b80c06 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -26,14 +26,18 @@ config DRM_ASAHI DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) config DRM_ASAHI_G15 - bool "Apple M3 Pro (G15S) support (experimental)" + bool "Apple M3 Pro/Max (G15S, G15C) support (experimental)" depends on DRM_ASAHI && ARM64 depends on APPLE_PMP=y || APPLE_PMP=DRM_ASAHI select CRYPTO_LIB_SHA256 + select RUST_FW_LOADER_ABSTRACTIONS help - Support for the G15S GPU of the T6030 MacBook Pros (J514S, J516S), - whose firmware (RTKit 2419) the boot loader leaves loaded. Only the - qualified firmware build is admitted. + Support for the G15S GPU of the T6030 MacBook Pros (J514S, J516S) + and the G15C GPU of the T6031 MacBook Pros (J514C, J516C), whose + firmware (RTKit 2419) the boot loader leaves loaded. Only the + qualified firmware builds are admitted. The T6031 board images are + a firmware file (asahi/t6031-j516c-board.bin), to be built in with + CONFIG_EXTRA_FIRMWARE. The G15 platform driver has no module alias: it binds when the asahi module is loaded explicitly (modprobe asahi), after the diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 0e500809826c8..298802a5da289 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -132,8 +132,8 @@ module! { description: "G15: completion repoll interval, us (25..1000)", }, g15_idle_standby_us: u32 { - default: 2500, - description: "G15: firmware idle standby time, us (Globals +0x7a8, 100..100000)", + default: 0, + description: "G15: firmware idle standby time override, us (Globals +0x7a8, 100..100000; 0: the device tree's, else the tuned value)", }, g15_idle_min_conf: u32 { default: 500, @@ -147,9 +147,33 @@ module! { default: 1, description: "G15: admit only known CDM/VDM stream records (0: bring-up only)", }, + g15_pb_max_blocks: u32 { + default: 3072, + description: "G15: parameter-buffer growth maximum given to the firmware, blocks of 128 KiB (64..4095; the mapped 4095 minus this answers partial renders)", + }, + g15_pb_blocks: u32 { + default: 64, + description: "G15: parameter-buffer blocks of 128 KiB posted at (re)initialization (64..4095; the heap is backed on demand in 16 MiB chunks)", + }, + g15_pb_reclaim: u32 { + default: 2, + description: "G15: parameter-buffer reclaim when the engine has been idle: 0 never, 1 re-initialize the PB at the next render, 2 also free the backing above the reserve", + }, + g15_pb_idle_ms: u32 { + default: 1000, + description: "G15: idle time before the parameter buffer is reclaimed, ms (at least 100)", + }, + g15_pb_reclaim_blocks: u32 { + default: 1024, + description: "G15: parameter-buffer blocks posted above which an idle engine reclaims (a desktop stays below 1024)", + }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", }, + g15_accept_text_prefix: u64 { + default: 0, + description: "G15: admit an unpinned GPU firmware build whose TEXT SHA-256 starts with these 8 bytes (bring-up only)", + }, }, } diff --git a/drivers/gpu/drm/asahi/g15/adt.rs b/drivers/gpu/drm/asahi/g15/adt.rs index 4595904e0bc20..9b9d4c93b8d4d 100644 --- a/drivers/gpu/drm/asahi/g15/adt.rs +++ b/drivers/gpu/drm/asahi/g15/adt.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! The boot loader's ADT, read from its phram export. +//! The GPU regions, firmware segments and performance states: from the boot loader's ADT +//! export (T6030), or from the GPU device tree node the boot loader fills (T6031). +//! +//! # T6030: the ADT export //! //! The GPU node that the PMP/ADT handoff (`drivers/soc/apple/j514s-pmp-handoff.c`) synthesizes //! has only `compatible = "apple,agx-t6030"`, `reg`/`reg-names` (`asc`, `sgx`), `mboxes`/ @@ -15,9 +18,19 @@ //! checked too) and `pre-loaded == 1`; `/chosen` `dram-base`/`dram-size`. //! Admission: every region page-aligned, inside DRAM, disjoint from System RAM and from each //! other; TEXT at [`super::hw::UPPER`], DATA right after it. +//! +//! # T6031: the device tree node +//! +//! The boot loader fills the kernel's GPU node from the machine's ADT: `memory-region` entries +//! `ttbs`, `pagetables`, `handoff` and `pagetables-l2` (the UAT regions), `fw-text` and +//! `fw-data` (the loaded segments, no-map), `apple,firmware-segment-vas` (their two VAs) and +//! the `operating-points-v2` table (one `opp-hz` per performance state, `opp-microvolt` with +//! one cell per cluster). The same admission applies, without the DRAM bounds, and the four UAT +//! regions may lie in System RAM: they are reserved-memory nodes without `no-map`, which the +//! kernel maps, so the driver reaches them through its own mapping ([`super::regs::Mapping`]). use core::ops::Range; -use kernel::{device, prelude::*}; +use kernel::{c_str, device, prelude::*}; use super::boot::glue; use super::hw; @@ -42,8 +55,17 @@ pub(crate) struct GpuAdt { pub(crate) perf_mhz: [u32; PERF_STATES], pub(crate) perf_mv: [u32; PERF_STATES], pub(crate) sram_mv: [u32; PERF_STATES], + /// Per-cluster core mV of each state, rounded up from the device tree's microvolts, how + /// many voltage cells the node has (1 on the ADT path), and the state's power in mW (0 on + /// the ADT path). + pub(crate) cluster_mv: [[u32; CLUSTERS]; PERF_STATES], + pub(crate) cluster_cells: usize, + pub(crate) perf_mw: [u32; PERF_STATES], } +/// Voltage cells the layout keeps per state. +pub(crate) const CLUSTERS: usize = 4; + /// A private copy of the ADT. struct Adt<'a> { data: &'a [u8], @@ -282,6 +304,40 @@ fn parse(dev: &device::Device, adt: &Adt<'_>) -> Result { return Err(EINVAL); }; // Structural admission: the ADT says where; this checks it is sane. + admit_regions(dev, ®ion, Some(dram_base..dram_base + dram_size), false)?; + let [ttbs, tables, handoff, shared_l2, text, data] = region; + let mut out = GpuAdt { + ttbs, + tables, + handoff, + shared_l2, + text, + data, + perf_mhz: [0; PERF_STATES], + perf_mv: [0; PERF_STATES], + sram_mv: [0; PERF_STATES], + cluster_mv: [[0; CLUSTERS]; PERF_STATES], + cluster_cells: 1, + perf_mw: [0; PERF_STATES], + }; + collect_perf(dev, adt, sgx, &mut out)?; + for i in 0..PERF_STATES { + out.cluster_mv[i][0] = out.perf_mv[i]; + } + Ok(out) +} + +/// Structural admission of the six regions (same order as [`REGION_NAMES`]): page-aligned, +/// addressable, the fixed UAT sizes, disjoint from each other, and either outside System RAM +/// (the T6030 carveouts) or, with `reserved_ram` and for the four UAT regions only, lying +/// entirely in System RAM as reserved memory the GPU node names (the T6031 device tree). The +/// firmware segments are always outside System RAM. +fn admit_regions( + dev: &device::Device, + region: &[Range; 6], + dram: Option>, + reserved_ram: bool, +) -> Result { let fixed = [ hw::TTBS_SIZE, hw::TABLES_SIZE, @@ -292,17 +348,25 @@ fn parse(dev: &device::Device, adt: &Adt<'_>) -> Result { let r = ®ion[i]; let size = r.end - r.start; let end = r.end.wrapping_sub(1); + let outside_dram = match &dram { + Some(dram) => r.start < dram.start || end > dram.end - 1, + None => false, + }; + let placed = if reserved_ram && i < 4 { + glue::ram_disjoint(r.start, size) || glue::ram_within(r.start, size) + } else { + glue::ram_disjoint(r.start, size) + }; if size == 0 || (r.start | size) & (hw::PAGE - 1) != 0 || end > (hw::PA_MASK | (hw::PAGE - 1)) - || r.start < dram_base - || end > dram_base + dram_size - 1 + || outside_dram || (i < 4 && size != fixed[i]) - || !glue::ram_disjoint(r.start, size) + || !placed { dev_err!( dev, - "ADT GPU region {} ({}) [mem {:#x}-{:#x}] not admissible\n", + "GPU region {} ({}) [mem {:#x}-{:#x}] not admissible\n", i, REGION_NAMES[i], r.start, @@ -312,11 +376,44 @@ fn parse(dev: &device::Device, adt: &Adt<'_>) -> Result { } for j in 0..i { if r.start < region[j].end && region[j].start < r.end { - dev_err!(dev, "ADT GPU regions {} and {} overlap\n", j, i); + dev_err!(dev, "GPU regions {} and {} overlap\n", j, i); return Err(EINVAL); } } } + Ok(()) +} + +/// The GPU node's regions, segments and operating points (T6031). +fn from_device_tree(dev: &device::Device) -> Result { + const NAMES: [&CStr; 6] = [ + c_str!("ttbs"), + c_str!("pagetables"), + c_str!("handoff"), + c_str!("pagetables-l2"), + c_str!("fw-text"), + c_str!("fw-data"), + ]; + let node = dev.of_node().ok_or(ENODEV)?; + let mut region: [Range; 6] = Default::default(); + for (i, name) in NAMES.iter().enumerate() { + let r = node + .reserved_mem_region_to_resource_byname(name) + .inspect_err(|_| dev_err!(dev, "GPU node has no {} memory region\n", REGION_NAMES[i]))?; + let start = r.start() as u64; + let size = r.size() as u64; + region[i] = start..start.checked_add(size).ok_or(EINVAL)?; + } + // Firmware ABI: TEXT at the base of the upper half, DATA right after it. + let vas: KVec = node + .get_property(c_str!("apple,firmware-segment-vas")) + .inspect_err(|_| dev_err!(dev, "GPU node has no apple,firmware-segment-vas\n"))?; + let text_size = region[4].end - region[4].start; + if vas.len() != 2 || vas[0] != hw::UPPER || vas[1] != vas[0] + text_size { + dev_err!(dev, "GPU firmware segment VAs are not the firmware ABI's\n"); + return Err(EINVAL); + } + admit_regions(dev, ®ion, None, true)?; let [ttbs, tables, handoff, shared_l2, text, data] = region; let mut out = GpuAdt { ttbs, @@ -328,13 +425,57 @@ fn parse(dev: &device::Device, adt: &Adt<'_>) -> Result { perf_mhz: [0; PERF_STATES], perf_mv: [0; PERF_STATES], sram_mv: [0; PERF_STATES], + cluster_mv: [[0; CLUSTERS]; PERF_STATES], + cluster_cells: 0, + perf_mw: [0; PERF_STATES], }; - collect_perf(dev, adt, sgx, &mut out)?; + // Operating points: MHz, the highest cluster voltage of each and the clusters' voltages + // (rounded up) and power for the hardware record; the SRAM voltage is not in the device + // tree. + let opps = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "GPU node has no operating points\n"))?; + let mut count = 0; + for opp in opps.children() { + let hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; + let uw: Option = opp.get_opt_property(c_str!("opp-microwatt"))?; + let mv = uv.iter().max().copied().unwrap_or(0) / 1000; + if uv.is_empty() || uv.len() > CLUSTERS || (count > 0 && uv.len() != out.cluster_cells) { + dev_err!(dev, "GPU operating point {} has {} voltage cells\n", count, uv.len()); + return Err(EINVAL); + } + if count < PERF_STATES { + out.perf_mhz[count] = (hz / 1_000_000) as u32; + out.perf_mv[count] = mv; + out.sram_mv[count] = mv; + for (c, v) in uv.iter().enumerate() { + out.cluster_mv[count][c] = v.div_ceil(1000); + } + out.cluster_cells = uv.len(); + out.perf_mw[count] = uw.unwrap_or(0) / 1000; + } + count += 1; + } + if count != PERF_STATES { + dev_err!( + dev, + "GPU node has {} operating points, not {}\n", + count, + PERF_STATES + ); + return Err(EINVAL); + } Ok(out) } -/// Reads and admits the GPU's ADT data (read-only). -pub(crate) fn read(dev: &device::Device) -> Result { +/// Reads and admits the GPU's regions, segments and performance states (read-only), from the +/// source the SoC configuration names. +pub(crate) fn read(dev: &device::Device, cfg: &super::Config) -> Result { + if cfg.platform == hw::Platform::DeviceTree { + return from_device_tree(dev); + } let (start, size) = glue::adt_phram()?; if size == 0 || size > 0x200000 || start & 0x3fff != 0 { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g15/board.rs b/drivers/gpu/drm/asahi/g15/board.rs index 35367611412a1..26262268b0d82 100644 --- a/drivers/gpu/drm/asahi/g15/board.rs +++ b/drivers/gpu/drm/asahi/g15/board.rs @@ -8,43 +8,212 @@ //! {0,1,2,3,4,6,8,10,12}; MHz at +0xb5c, +0xf9c, +0x1350, core mV +0xb9c, SRAM mV +0xd9c, top //! MHz +0x4dc4). The tables in [`data`] are generated, and every expanded image is checked //! against its hash at boot. +//! +//! A SoC whose board data comes from its device tree ([`super::hw::Platform::DeviceTree`]) gets +//! its hardware record from [`hardware_record`] ([`record`]) and its Globals and Power from +//! [`device_tree`]: the G15S templates with the GPU node's power words and the hardware +//! record's maximum power ([`words`]). +pub(crate) mod blob; pub(crate) mod data; +pub(crate) mod record; +pub(crate) mod template; +pub(crate) mod words; -use kernel::{device, prelude::*}; +use kernel::{c_str, device, prelude::*}; -use super::adt::GpuAdt; +use super::adt::{self, GpuAdt}; use super::boot::glue; +use super::fw::initdata::{globals, hwdata, power}; +use super::hw::{self, va}; +pub(crate) use template::{Line, Template}; + +static_assert!(words::GLOBALS_SIZE == globals::SIZE); +static_assert!(words::POWER_SIZE == power::SIZE); +static_assert!(words::HWDATA_A == hwdata::HWDATA_A); +static_assert!(words::HWDATA_A_MAX_POWER == hwdata::MAX_POWER_A); +static_assert!(words::TOP_STATE as usize == DVFS_STATES - 1); +static_assert!(record::HWDATA_A == hwdata::HWDATA_A); +static_assert!(record::MAX_POWER == hwdata::HWDATA_A + hwdata::MAX_POWER_A); +static_assert!(record::IOMAPS == hwdata::IOMAPS); +static_assert!(record::PERF_STATES == adt::PERF_STATES); +static_assert!(record::CLUSTERS == adt::CLUSTERS); -/// One template line: 16 bytes at `offset`. -pub(crate) struct Line { - pub(crate) offset: u32, - pub(crate) data: [u8; 16], +/// Expands a template into `out` and verifies its hash. +pub(crate) fn expand(template: &Template, out: &mut [u8]) -> Result { + template::fill(template, out).map_err(|_| EINVAL)?; + if glue::sha256(&out[..template.size]) != template.sha256 { + return Err(EINVAL); + } + Ok(()) } -/// A sparse template. -pub(crate) struct Template { - pub(crate) size: usize, - pub(crate) lines: &'static [Line], - pub(crate) sha256: [u8; 32], +/// The GPU node's power words ([`words::PROPS`] and [`words::ZONES`]). +fn power_words(dev: &device::Device) -> Result { + let node = dev.of_node().ok_or(ENODEV)?; + let mut values = [0u32; words::PROPS.len()]; + for (value, name) in values.iter_mut().zip(words::PROPS.iter()) { + *value = node + .get_property(name) + .inspect_err(|_| dev_err!(dev, "GPU node has no u32 {}\n", name))?; + } + let zones: KVec = node.get_opt_property(words::ZONES)?.unwrap_or_default(); + words::PowerWords::new(values, &zones).map_err(|why| { + dev_err!(dev, "GPU node power words: {}\n", why); + EINVAL + }) } -/// Expands a template into `out` and verifies its hash. -pub(crate) fn expand(template: &Template, out: &mut [u8]) -> Result { - let out = out.get_mut(..template.size).ok_or(EINVAL)?; - out.fill(0); - for line in template.lines { - let offset = line.offset as usize; - if offset >= template.size { +/// A CS or AFR operating-point table of the GPU node (`name`): MHz and one voltage cell per +/// state, at most `max` states. +fn aux_opps( + dev: &device::Device, + node: &kernel::of::Node, + name: &CStr, + out: &mut [record::Opp], +) -> Result { + let table = node + .parse_phandle(name, 0) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "GPU node has no {}\n", name))?; + let mut n = 0; + for opp in table.children() { + let hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; + if uv.len() != 1 || n >= out.len() { + dev_err!( + dev, + "GPU {} table: state {} has {} cells\n", + name, + n, + uv.len() + ); return Err(EINVAL); } - let n = 16.min(template.size - offset); - out[offset..offset + n].copy_from_slice(&line.data[..n]); + out[n] = record::Opp { + mhz: (hz / 1_000_000) as u32, + mv: [uv[0].div_ceil(1000), 0, 0, 0], + cells: 1, + mw: 0, + }; + n += 1; } - if glue::sha256(out) != template.sha256 { + if n == 0 { + dev_err!(dev, "GPU {} table is empty\n", name); return Err(EINVAL); } - Ok(()) + Ok(n) +} + +/// The hardware record of a device-tree SoC ([`record`]): built from the GPU node's properties +/// and operating-point tables, the layout's IOMapping table and the ADT regions, with the +/// words the device tree does not carry copied from `file`, the machine's own record. +pub(crate) fn hardware_record( + dev: &device::Device, + cfg: &super::Config, + adt: &GpuAdt, + file: &[u8], +) -> Result> { + let layout = cfg.layout; + let node = dev.of_node().ok_or(ENODEV)?; + let mut props = [0u32; record::PROPS.len()]; + for (value, (name, default)) in props.iter_mut().zip(record::PROPS.iter()) { + *value = match node.get_opt_property(name)? { + Some(v) => v, + None => default + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "GPU node has no u32 {}\n", name))?, + }; + } + let zones: KVec = node.get_opt_property(words::ZONES)?.unwrap_or_default(); + let mut main = [record::Opp::default(); record::PERF_STATES]; + for (i, o) in main.iter_mut().enumerate() { + *o = record::Opp { + mhz: adt.perf_mhz[i], + mv: adt.cluster_mv[i], + cells: adt.cluster_cells, + mw: adt.perf_mw[i], + }; + } + let mut cs = [record::Opp::default(); 16]; + let mut afr = [record::Opp::default(); 8]; + let ncs = aux_opps(dev, &node, c_str!("apple,cs-opp"), &mut cs)?; + let nafr = aux_opps(dev, &node, c_str!("apple,afr-opp"), &mut afr)?; + if layout.record_slots.len() != layout.iomaps.len() { + return Err(EINVAL); + } + let mut slots = [record::IoSlot::default(); record::IOMAP_SLOTS]; + for (k, (map, &(total, element, writable))) in + layout.iomaps.iter().zip(layout.record_slots).enumerate() + { + slots[k] = record::IoSlot { + slot: map.slot, + pa: map.pa + map.delta, + va: map.va + map.delta, + total, + element, + writable, + }; + } + let max_power_mw = u32::from_le_bytes( + file.get(record::MAX_POWER..record::MAX_POWER + 4) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + ); + let inputs = record::Inputs { + props, + zones: &zones, + main: &main, + cs: &cs[..ncs], + afr: &afr[..nafr], + slots: &slots[..layout.iomaps.len()], + empty_writable: layout.record_empty_writable, + ttbat: adt.ttbs.start, + usc_base: hw::client::COMPACT_WINDOW.start, + user_end: hw::USER_RANGE.end, + timestamps: va::TIMESTAMPS, + max_power_mw, + }; + let mut out = KVVec::from_elem(0u8, record::SIZE, GFP_KERNEL)?; + record::from_file(&mut out, file) + .and_then(|()| record::build(&mut out, &inputs)) + .map_err(|why| { + dev_err!(dev, "hardware record from the device tree: {}\n", why); + EINVAL + })?; + Ok(out) +} + +/// Globals and Power of a device-tree SoC: the verified G15S templates with the GPU node's +/// power words and the maximum power of `hardware`, this machine's hardware record. Returns the +/// two images and that maximum power in mW. +pub(crate) fn device_tree( + dev: &device::Device, + hardware: &[u8], +) -> Result<(KVVec, KVVec, u32)> { + let w = power_words(dev)?; + let at = hwdata::HWDATA_A + hwdata::MAX_POWER_A; + let max_power_mw = u32::from_le_bytes( + hardware + .get(at..at + 4) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + ); + let mut glob = KVVec::from_elem(0u8, globals::SIZE, GFP_KERNEL)?; + let mut pwr = KVVec::from_elem(0u8, power::SIZE, GFP_KERNEL)?; + if expand(&data::GLOBALS, &mut glob).is_err() || expand(&data::POWER, &mut pwr).is_err() { + dev_err!(dev, "board data template hash mismatch\n"); + return Err(EINVAL); + } + words::globals(&mut glob, max_power_mw, &w) + .and_then(|()| words::power(&mut pwr, max_power_mw, &w)) + .map_err(|why| { + dev_err!(dev, "board data from the device tree: {}\n", why); + EINVAL + })?; + Ok((glob, pwr, max_power_mw)) } /// DVFS states of the firmware tables. @@ -52,6 +221,12 @@ pub(crate) const DVFS_STATES: usize = 9; const DVFS_A: [usize; DVFS_STATES] = [0, 1, 2, 3, 5, 7, 9, 11, 13]; const DVFS_B: [usize; DVFS_STATES] = [0, 1, 2, 3, 4, 6, 8, 10, 12]; +/// Table A's frequency of each firmware DVFS state, MHz: the ADT (or device-tree) operating +/// points the firmware's state indices name. +pub(crate) fn table_a_mhz(adt: &GpuAdt) -> [u32; DVFS_STATES] { + DVFS_A.map(|i| adt.perf_mhz[i]) +} + fn put32(hw: &mut [u8], offset: usize, value: u32) -> Result { hw.get_mut(offset..offset + 4) .ok_or(EINVAL)? diff --git a/drivers/gpu/drm/asahi/g15/board/blob.rs b/drivers/gpu/drm/asahi/g15/board/blob.rs new file mode 100644 index 0000000000000..04cff8bbaa3d9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/board/blob.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Board images from a firmware file. +//! +//! A SoC whose board images are not built-in templates names a firmware file +//! ([`super::super::hw::Board::Blob`]), built into the kernel with `CONFIG_EXTRA_FIRMWARE` or +//! loaded from `/lib/firmware`, and pins its SHA-256. The file holds the three images as the +//! firmware of that machine accepted them; the build then writes the pointers and policy words +//! it owns into them (TTBAT, IOMapping VAs, the timestamp arena, the idle policy). +//! +//! Layout, little-endian: the magic `T6031BRD`, u32 format version (1), u32 image count (3), +//! then per image `u32 kind, u32 size` with kinds 0 (HardwareData), 1 (Globals) and +//! 2 (Power), then the images' bytes in that order. Sizes must match the SoC's layout. + +use kernel::{device, firmware::Firmware, prelude::*}; + +use super::super::boot::glue; +use super::super::identity::Hex; + +const MAGIC: &[u8; 8] = b"T6031BRD"; +const VERSION: u32 = 1; +const IMAGES: usize = 3; + +/// The three images. +pub(crate) struct Images { + pub(crate) hardware: KVVec, + pub(crate) globals: KVVec, + pub(crate) power: KVVec, +} + +fn le32(b: &[u8], off: usize) -> Result { + Ok(u32::from_le_bytes( + b.get(off..off + 4) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +fn copy(b: &[u8], range: core::ops::Range) -> Result> { + let src = b.get(range).ok_or(EINVAL)?; + let mut out = KVVec::from_elem(0u8, src.len(), GFP_KERNEL)?; + out.copy_from_slice(src); + Ok(out) +} + +/// Requests `name`, verifies its digest against `sha256` and splits it into the images, each +/// of the size `sizes` gives for its kind. +pub(crate) fn load( + dev: &device::Device, + name: &CStr, + sha256: &[u8; 32], + sizes: [usize; IMAGES], +) -> Result { + let fw = Firmware::request(name, dev).inspect_err(|e| { + dev_err!(dev, "board images {}: not available ({:?})\n", name, e) + })?; + let data = fw.data(); + let digest = glue::sha256(data); + if digest != *sha256 { + dev_err!( + dev, + "board images {}: sha256 {} is not the pinned one\n", + name, + Hex(&digest) + ); + return Err(EINVAL); + } + if data.len() < 16 || &data[..8] != MAGIC || le32(data, 8)? != VERSION { + dev_err!(dev, "board images {}: unknown header\n", name); + return Err(EINVAL); + } + if le32(data, 12)? as usize != IMAGES { + dev_err!(dev, "board images {}: not {} images\n", name, IMAGES); + return Err(EINVAL); + } + let mut at = 16 + IMAGES * 8; + let mut images: [Option>; IMAGES] = [None, None, None]; + for i in 0..IMAGES { + let kind = le32(data, 16 + i * 8)? as usize; + let size = le32(data, 16 + i * 8 + 4)? as usize; + if kind != i || size != sizes[i] { + dev_err!( + dev, + "board images {}: image {} is kind {} of {} bytes, need {}\n", + name, + i, + kind, + size, + sizes[i] + ); + return Err(EINVAL); + } + images[i] = Some(copy(data, at..at + size)?); + at += size; + } + if at != data.len() { + dev_err!(dev, "board images {}: trailing bytes\n", name); + return Err(EINVAL); + } + let [hardware, globals, power] = images; + dev_info!( + dev, + "board images {}: sha256 {} accepted\n", + name, + Hex(&digest) + ); + Ok(Images { + hardware: hardware.ok_or(EINVAL)?, + globals: globals.ok_or(EINVAL)?, + power: power.ok_or(EINVAL)?, + }) +} diff --git a/drivers/gpu/drm/asahi/g15/board/record.rs b/drivers/gpu/drm/asahi/g15/board/record.rs new file mode 100644 index 0000000000000..15cce354b0f01 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/board/record.rs @@ -0,0 +1,1305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The J516C hardware record (0xb480 bytes) from the device tree. +//! +//! The record is HardwareDataB (identity, IOMapping records, the DVFS tables) at 0, HardwareDataA +//! (the power, performance and thermal controllers) at [`HWDATA_A`], and the G15C tail (sensor +//! keys). Every word is one of: +//! +//! - a device-tree value or a function of one ([`PROPS`], the operating-point tables, the +//! reserved `ttbs` region): the controller gains, time constants and their derived filter +//! words (`(tc - 1) / tc`, `1 / tc`, gain x period, periods x 4, periods x period, clocks), +//! the two voltage-sorted DVFS tables with per-cluster voltages, the CS and AFR tables, the +//! state cap and base state; +//! - a layout constant of the 2419 firmware's G15C record: flags, the IOMapping flag words, +//! the identity words, fixed factors (65536, 95, 100, 1.02, 0.8/0.2, 625, 11000, 125, 5); +//! - a word the device tree does not carry ([`FROM_FILE`]), copied from the machine's board file +//! by [`from_file`] and listed with its rule in the project notes: the maximum power +//! (HardwareDataA +0x6a4; every other maximum-power word derives from it), the MTR sensor +//! masks, the relative-power table (the device tree's operating-point powers are a boot-loader +//! stand-in) and the tail's six sensor rows; +//! - a word the file carries and the firmware was shown not to need ([`UNNEEDED`]): zero. +//! +//! HardwareDataA has the G15S shape of the T6030 templates with insertions ([`a`]): 8 bytes +//! before the fast-die controller (+0x8bc), 16-wide instead of 8-wide coefficient rows (+0x200 +//! twice), 16-byte instead of 8-byte sensor masks (+8 twice), 0x600 and 0x2000 bytes of zero +//! blocks further on. The driver still writes the TTBAT, the IOMapping VAs, the timestamp arena +//! and the initdata timestamp as it does for the templates. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. + +use core::ffi::CStr; + +use super::words::{f32_div, f32_from_u32, f32_mul}; + +pub(crate) const SIZE: usize = 0xb480; +pub(crate) const HWDATA_A: usize = 0x4580; +/// IOMapping records: 32 bytes per slot. +pub(crate) const IOMAPS: usize = 0x640; +pub(crate) const IOMAP_SLOTS: usize = 31; +/// DVFS table entries. +pub(crate) const STATES: usize = 16; +/// Entries of the CS and AFR tables. +const CS_STATES: usize = 16; +const AFR_STATES: usize = 8; +/// Operating points of the layout and voltage lanes per state. +pub(crate) const PERF_STATES: usize = 14; +pub(crate) const CLUSTERS: usize = 4; +const LANES: usize = 8; +/// The AIC timer base clock, kHz. +const BASE_CLOCK_KHZ: u32 = 24000; + +// IEEE single bit patterns of the record's fixed factors. +const F_1: u32 = 0x3f800000; +const F_0_2: u32 = 0x3e4ccccd; +const F_0_8: u32 = 0x3f4ccccd; +const F_0_95: u32 = 0x3f733333; +const F_1_02: u32 = 0x3f828f5c; +const F_5: u32 = 0x40a00000; +const F_95: u32 = 0x42be0000; +const F_100: u32 = 0x42c80000; +const F_8000: u32 = 0x45fa0000; +const F_65536: u32 = 0x47800000; + +/// The GPU node's properties the record takes, each one u32 (gains are IEEE single bits), with +/// the value a node without the property gets (`None`: required). +pub(crate) const PROPS: [(&CStr, Option); 43] = [ + (c"apple,power-sample-period", None), + (c"apple,perf-base-pstate", None), + (c"apple,min-sram-microvolt", None), + (c"apple,csafr-min-sram-microvolt", None), + (c"apple,avg-power-filter-tc-ms", None), + (c"apple,avg-power-ki-only", None), + (c"apple,avg-power-kp", None), + (c"apple,avg-power-min-duty-cycle", None), + (c"apple,avg-power-target-filter-tc", None), + (c"apple,fast-die0-integral-gain", None), + (c"apple,fast-die0-proportional-gain", None), + (c"apple,fast-die0-prop-tgt-delta", Some(0)), + (c"apple,perf-boost-ce-step", None), + (c"apple,perf-boost-min-util", None), + (c"apple,perf-filter-drop-threshold", None), + (c"apple,perf-filter-time-constant", None), + (c"apple,perf-filter-time-constant2", None), + (c"apple,perf-integral-gain", None), + (c"apple,perf-integral-gain2", None), + (c"apple,perf-integral-min-clamp", None), + (c"apple,perf-proportional-gain", None), + (c"apple,perf-proportional-gain2", None), + (c"apple,perf-reset-iters", Some(6)), + (c"apple,perf-tgt-utilization", None), + (c"apple,ppm-filter-time-constant-ms", None), + (c"apple,ppm-ki", None), + (c"apple,ppm-kp", None), + (c"apple,pwr-filter-time-constant", None), + (c"apple,pwr-integral-gain", None), + (c"apple,pwr-integral-min-clamp", None), + (c"apple,pwr-min-duty-cycle", None), + (c"apple,pwr-proportional-gain", None), + (c"apple,pwr-sample-period-aic-clks", None), + (c"apple,se-engagement-criteria", None), + (c"apple,se-filter-time-constant", None), + (c"apple,se-filter-time-constant-1", None), + (c"apple,se-inactive-threshold", None), + (c"apple,se-ki", None), + (c"apple,se-ki-1", None), + (c"apple,se-kp", None), + (c"apple,se-kp-1", None), + (c"apple,se-reset-criteria", None), + (c"apple,se-target", None), +]; +const PERIOD_MS: usize = 0; +const BASE_PSTATE: usize = 1; +const MIN_SRAM_UV: usize = 2; +const CSAFR_MIN_SRAM_UV: usize = 3; +const AVG_TC_MS: usize = 4; +const AVG_KI: usize = 5; +const AVG_KP: usize = 6; +const AVG_MIN_DUTY: usize = 7; +const AVG_TGT_TC: usize = 8; +const FD_KI: usize = 9; +const FD_KP: usize = 10; +const FD_DELTA: usize = 11; +const BOOST_CE: usize = 12; +const BOOST_MIN: usize = 13; +const PERF_DROP: usize = 14; +const PERF_TC: usize = 15; +const PERF_TC2: usize = 16; +const PERF_KI: usize = 17; +const PERF_KI2: usize = 18; +const PERF_MIN_CLAMP: usize = 19; +const PERF_KP: usize = 20; +const PERF_KP2: usize = 21; +const PERF_RESET: usize = 22; +const PERF_TGT: usize = 23; +const PPM_TC_MS: usize = 24; +const PPM_KI: usize = 25; +const PPM_KP: usize = 26; +const PWR_TC: usize = 27; +const PWR_KI: usize = 28; +const PWR_MIN_CLAMP: usize = 29; +const PWR_MIN_DUTY: usize = 30; +const PWR_KP: usize = 31; +const AIC_CLKS: usize = 32; +const SE_CRIT: usize = 33; +const SE_TC: usize = 34; +const SE_TC1: usize = 35; +const SE_INACT: usize = 36; +const SE_KI: usize = 37; +const SE_KI1: usize = 38; +const SE_KP: usize = 39; +const SE_KP1: usize = 40; +const SE_RESET: usize = 41; +const SE_TARGET: usize = 42; + +/// One operating point: MHz, the clusters' voltages in mV (rounded up from microvolts), how +/// many voltage cells the node had, and mW. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Opp { + pub(crate) mhz: u32, + pub(crate) mv: [u32; CLUSTERS], + pub(crate) cells: usize, + pub(crate) mw: u32, +} + +impl Opp { + fn max_mv(&self) -> u32 { + self.mv[..self.cells].iter().copied().max().unwrap_or(0) + } +} + +/// One IOMapping record: slot, physical address and firmware VA (both with the in-page offset), +/// mapped bytes, element bytes, writable. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct IoSlot { + pub(crate) slot: u8, + pub(crate) pa: u64, + pub(crate) va: u64, + pub(crate) total: u32, + pub(crate) element: u32, + pub(crate) writable: bool, +} + +/// Everything the record is built from. +pub(crate) struct Inputs<'a> { + pub(crate) props: [u32; PROPS.len()], + /// `apple,power-zones` cells (triples). + pub(crate) zones: &'a [u32], + pub(crate) main: &'a [Opp], + pub(crate) cs: &'a [Opp], + pub(crate) afr: &'a [Opp], + pub(crate) slots: &'a [IoSlot], + /// Unmapped slots whose flag word the record nevertheless holds as 1. + pub(crate) empty_writable: &'a [u8], + /// The TTBAT page (the `ttbs` region). + pub(crate) ttbat: u64, + /// Firmware VAs the record repeats: the USC base, the page past the user range, the + /// timestamp arena. + pub(crate) usc_base: u64, + pub(crate) user_end: u64, + pub(crate) timestamps: u64, + /// The maximum GPU power in mW ([`MAX_POWER`], a word the device tree does not carry). + pub(crate) max_power_mw: u32, +} + +/// HardwareDataA's maximum-power word (= `fw::initdata::hwdata::MAX_POWER_A`). +pub(crate) const MAX_POWER: usize = HWDATA_A + 0x6a4; + +/// Words the device tree does not carry, copied from the board file. Record offsets. +#[rustfmt::skip] +pub(crate) const FROM_FILE: [usize; 32] = [ + // The relative powers of states 1..7 (state 8 is 100): the device tree's operating-point + // powers are a boot-loader stand-in. + 0x1060, 0x1064, 0x1068, 0x106c, 0x1070, 0x1074, 0x1078, + // HardwareDataA: the maximum power, the MTR fast-die masks (+0x8ac, +0x1490) and the alarm + // mask (+0x24a8); the device tree's apple,fast-die0-sensor-mask is another value. + MAX_POWER, + HWDATA_A + 0x8ac, HWDATA_A + 0x8b0, + HWDATA_A + 0x1490, HWDATA_A + 0x1494, + HWDATA_A + 0x24a8, HWDATA_A + 0x24ac, + // The tail's six sensor rows (key, id, kind; 12 bytes each; the boot loader does not export + // the aon-ptd table). + HWDATA_A + 0x6c58, HWDATA_A + 0x6c5c, HWDATA_A + 0x6c60, + HWDATA_A + 0x6c64, HWDATA_A + 0x6c68, HWDATA_A + 0x6c6c, + HWDATA_A + 0x6c70, HWDATA_A + 0x6c74, HWDATA_A + 0x6c78, + HWDATA_A + 0x6c7c, HWDATA_A + 0x6c80, HWDATA_A + 0x6c84, + HWDATA_A + 0x6c88, HWDATA_A + 0x6c8c, HWDATA_A + 0x6c90, + HWDATA_A + 0x6c94, HWDATA_A + 0x6c98, HWDATA_A + 0x6c9c, +]; + +/// Words the board file carries that the record leaves zero: the firmware ran, passed every +/// check and reached its top state without them (one boot each). The cross-check accepts the +/// file's value there. 27 words in the HardwareDataB +0x38..+0x638 block and two all-ones words +/// after it (zero in the G15S templates); the tail's thirteen readings (a single at +/// HardwareDataA +0x6d34, twelve singles after +0x6d80). +#[rustfmt::skip] +pub(crate) const UNNEEDED: [usize; 42] = [ + 0xe0, 0xe8, 0xf4, 0xf8, 0xfc, 0x100, 0x104, 0x108, 0x10c, 0x2d8, 0x2dc, 0x2e0, 0x2e4, + 0x2e8, 0x2ec, 0x3e0, 0x3e8, 0x3f4, 0x3f8, 0x400, 0x40c, 0x5d8, 0x5dc, 0x5e0, 0x5e4, + 0x5e8, 0x5ec, 0x30d8, 0x30f0, + HWDATA_A + 0x6d34, + HWDATA_A + 0x6d8c, HWDATA_A + 0x6d90, HWDATA_A + 0x6d94, HWDATA_A + 0x6d98, + HWDATA_A + 0x6dcc, HWDATA_A + 0x6dd0, HWDATA_A + 0x6dd4, HWDATA_A + 0x6dd8, + HWDATA_A + 0x6dec, HWDATA_A + 0x6df4, HWDATA_A + 0x6dfc, HWDATA_A + 0x6e04, +]; + +/// The tail's sensor rows: six rows of {key, id, kind} from [`SENSOR_ROWS`]; the keys of the +/// rows whose kind byte is `g`, their presence mask, then the `r` and `f` rows' keys with a +/// count of one each. +const SENSOR_ROWS: usize = HWDATA_A + 0x6c58; +const SENSOR_ROW_COUNT: usize = 6; +const SENSOR_G_KEYS: usize = HWDATA_A + 0x6cdc; +const SENSOR_G_MASK: usize = HWDATA_A + 0x6cfc; +const SENSOR_R_KEY: usize = HWDATA_A + 0x6d04; +const SENSOR_R_COUNT: usize = HWDATA_A + 0x6d0c; +const SENSOR_F_KEY: usize = HWDATA_A + 0x6d14; +const SENSOR_F_COUNT: usize = HWDATA_A + 0x6d18; + +type R = Result; + +fn get32(b: &[u8], at: usize) -> R { + let w = b.get(at..at + 4).ok_or("record too short")?; + Ok(u32::from_le_bytes([w[0], w[1], w[2], w[3]])) +} + +fn put32(b: &mut [u8], at: usize, v: u32) -> R<()> { + b.get_mut(at..at + 4) + .ok_or("record too short")? + .copy_from_slice(&v.to_le_bytes()); + Ok(()) +} + +fn put64(b: &mut [u8], at: usize, v: u64) -> R<()> { + b.get_mut(at..at + 8) + .ok_or("record too short")? + .copy_from_slice(&v.to_le_bytes()); + Ok(()) +} + +/// `1 / n` as a single. +fn inv(n: u32) -> R { + f32_div(F_1, f32_from_u32(n)).ok_or("a time constant is 0") +} + +/// `(n - 1) / n` as a single (the filter's complement, `1 - 1/n`). +fn complement(n: u32) -> R { + if n == 0 { + return Err("a time constant is 0"); + } + f32_div(f32_from_u32(n - 1), f32_from_u32(n)).ok_or("a time constant is 0") +} + +/// The G15C HardwareDataA offset of a G15S-layout word. +pub(crate) const fn a(off: usize) -> usize { + let shifted = if off < 0x8bc { + off + } else if off < 0x970 { + off + 8 + } else if off < 0xb70 { + // The coefficient tables, 16 instead of 8 singles per row: not written here. + off + 8 + } else if off < 0x1290 { + off + 0x208 + } else if off < 0x14b4 { + off + 0x210 + } else if off < 0x1aa0 { + off + 0xa10 + } else if off < 0x3aa4 { + off + 0xa18 + } else { + off + 0x2a18 + }; + HWDATA_A + shifted +} + +/// The two voltage-sorted DVFS tables of the operating points: entry 0 is state 0 (off), then +/// per distinct maximum voltage, ascending, the highest-frequency state (table A, `primary`) +/// and the lowest-frequency one (table B, `secondary`). States sharing a voltage must share +/// their cluster voltages; frequencies must rise along both tables and every cluster's voltage +/// along table A. +pub(crate) struct Tables { + pub(crate) len: usize, + pub(crate) primary: [u8; STATES], + pub(crate) secondary: [u8; STATES], +} + +impl Tables { + pub(crate) fn top(&self) -> usize { + self.len - 1 + } +} + +pub(crate) fn tables(main: &[Opp]) -> R { + if main.len() < 2 || main.len() > u8::MAX as usize { + return Err("operating points: fewer than 2 or more than 255"); + } + if main[0].mhz != 0 || main[1..].iter().any(|s| s.mhz == 0) { + return Err("operating points: state 0 is not the only 0 MHz state"); + } + let mut t = Tables { + len: 1, + primary: [0; STATES], + secondary: [0; STATES], + }; + let mut last_mv = 0; + loop { + let Some(mv) = main[1..] + .iter() + .map(Opp::max_mv) + .filter(|&mv| mv > last_mv) + .min() + else { + break; + }; + let group = (1..main.len()).filter(|&i| main[i].max_mv() == mv); + let (mut first, mut hi, mut lo) = (0usize, 0usize, 0usize); + for i in group { + if first == 0 { + first = i; + } + if hi == 0 || main[i].mhz > main[hi].mhz { + hi = i; + } + if lo == 0 || main[i].mhz < main[lo].mhz { + lo = i; + } + if main[i].mv != main[first].mv { + return Err("operating points at one voltage differ per cluster"); + } + } + if t.len >= STATES { + return Err("operating points: more than 16 voltages"); + } + let (pp, ps) = ( + &main[t.primary[t.len - 1] as usize], + &main[t.secondary[t.len - 1] as usize], + ); + if main[hi].mhz <= pp.mhz || main[lo].mhz <= ps.mhz { + return Err("operating points: frequency does not rise with voltage"); + } + if t.len > 1 && (0..main[hi].cells).any(|c| main[hi].mv[c] <= pp.mv[c]) { + return Err("operating points: a cluster's voltage does not rise with the table"); + } + t.primary[t.len] = hi as u8; + t.secondary[t.len] = lo as u8; + t.len += 1; + last_mv = mv; + } + if t.len < 2 { + return Err("operating points: no usable state"); + } + Ok(t) +} + +/// A voltage lane of a state: the cluster's voltage, or cluster 0's past the node's cells. +fn lane(s: &Opp, j: usize) -> u32 { + if j < s.cells { + s.mv[j] + } else { + s.mv[0] + } +} + +fn hwdata_b(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { + let p = &i.props; + put64(out, 0x0, 0x6f_0000_0000)?; + put64(out, 0x8, 0xffc0_0000)?; + put64(out, 0x10, i.usc_base)?; + put64(out, 0x18, i.usc_base)?; + put64(out, 0x20, i.user_end)?; + put64(out, 0x28, i.timestamps)?; + for s in i.slots { + if s.slot as usize >= IOMAP_SLOTS { + return Err("IOMapping slot past the table"); + } + let r = IOMAPS + 32 * s.slot as usize; + put64(out, r, s.pa)?; + put64(out, r + 8, s.va)?; + put32(out, r + 0x10, s.total)?; + put32(out, r + 0x14, s.element)?; + put32(out, r + 0x18, s.writable as u32)?; + } + for &slot in i.empty_writable { + if slot as usize >= IOMAP_SLOTS || i.slots.iter().any(|s| s.slot == slot) { + return Err("an empty-writable slot is mapped or past the table"); + } + put32(out, IOMAPS + 32 * slot as usize + 0x18, 1)?; + } + // Identity and flags. + put32(out, 0xa28, 0x6031)?; + put32(out, 0xa2c, 1)?; + put32(out, 0xa30, 2)?; + for at in [0xa38, 0xa3c, 0xa40, 0xa50, 0xa58, 0xa60] { + put32(out, at, 1)?; + } + put32(out, 0xa68, BASE_CLOCK_KHZ)?; + put32(out, 0xa6c, 1)?; + put32(out, 0xa70, p[PERIOD_MS])?; + put32(out, 0xa78, 1)?; + put64(out, 0xa7c, 0x1_0000_0001)?; + put32(out, 0xa9c, 0x1f)?; + put64(out, 0xabc, 4)?; + put32(out, 0xacc, 1)?; + put32(out, 0xad0, 1)?; + put32(out, 0xae4, 0x31)?; + put32(out, 0xb04, 1)?; + put32(out, 0xb20, 0x28)?; + put32(out, 0xb24, 0xff)?; + put32(out, 0xb40, 1)?; + put64(out, 0xb44, i.ttbat)?; + // GPU core 24 (G15C), revision 4, 40 core slots. + put32(out, 0xb4c, 24)?; + put32(out, 0xb50, 4)?; + put32(out, 0xb54, 40)?; + put32(out, 0xb58, t.top() as u32)?; + let min_sram = p[MIN_SRAM_UV].div_ceil(1000); + let top = &i.main[t.primary[t.top()] as usize]; + let base_index = p[BASE_PSTATE] as usize; + let base = i + .main + .get(base_index) + .ok_or("perf-base-pstate past the states")?; + if t.primary[base_index] as usize != base_index { + return Err("perf-base-pstate is not its own table-A entry"); + } + if top.mhz <= base.mhz { + return Err("the top state is not above the base state"); + } + for k in 0..t.len { + let ps = &i.main[t.primary[k] as usize]; + let ss = &i.main[t.secondary[k] as usize]; + put32(out, 0xb5c + 4 * k, ps.mhz)?; + put32(out, 0xf9c + 4 * k, ss.mhz)?; + put32(out, 0x1350 + 4 * k, ss.mhz)?; + for j in 0..LANES { + let v = lane(ps, j); + put32(out, 0xb9c + 0x20 * k + 4 * j, v)?; + put32(out, 0xd9c + 0x20 * k + 4 * j, v.max(min_sram))?; + } + for j in 0..ss.cells { + put32(out, 0x1390 + 0x20 * k + 4 * j, ss.mv[j])?; + put32(out, 0x1590 + 0x20 * k + 4 * j, ss.mv[j].max(min_sram))?; + } + put32(out, 0xfdc + 4 * k, F_1_02)?; + put32(out, 0x10dc + 4 * k, t.primary[k] as u32)?; + put32(out, 0x111c + 4 * k, t.secondary[k] as u32)?; + if k > 0 { + // Relative power (a FROM_FILE word on the J516C) and relative boost frequency. + if top.mw != 0 { + put32( + out, + 0x105c + 4 * k, + (100 * u64::from(ps.mw) / u64::from(top.mw)) as u32, + )?; + } + if ps.mhz > base.mhz { + let span = u64::from(top.mhz - base.mhz); + put32( + out, + 0x109c + 4 * k, + (100 * u64::from(ps.mhz - base.mhz) / span) as u32, + )?; + } + } + } + put32(out, 0x134c, t.top() as u32)?; + // The CS and AFR tables: the top state twice, frequencies, voltages, SRAM voltages. + let cs_sram = p[CSAFR_MIN_SRAM_UV] / 1000; + let mut aux = |base: usize, tab: &[Opp], n: usize| -> R { + if tab.is_empty() || tab.len() > n { + return Err("CS/AFR table empty or longer than its layout"); + } + let top = (tab.len() - 1) as u32; + put32(out, base, top)?; + put32(out, base + 4, top)?; + let (fr, vo, sr) = (base + 8, base + 8 + 4 * n, base + 8 + 12 * n); + for (k, s) in tab.iter().enumerate() { + put32(out, fr + 4 * k, s.mhz)?; + for j in 0..s.cells.min(2) { + put32(out, vo + 8 * k + 4 * j, s.mv[j])?; + put32(out, sr + 8 * k + 4 * j, s.mv[j].max(cs_sram))?; + } + } + Ok(sr + 8 * n) + }; + let end = aux(0x115c, i.cs, CS_STATES)?; + if end != 0x12a4 { + return Err("CS table layout"); + } + let end = aux(0x12a4, i.afr, AFR_STATES)?; + if end != 0x134c { + return Err("AFR table layout"); + } + put32(out, 0x17b8, 5)?; + put32(out, 0x17c0, 3)?; + put32(out, 0x17c4, 7)?; + put32(out, 0x17c8, 7)?; + for at in [ + 0x17e8, 0x17fc, 0x1800, 0x1804, 0x1808, 0x1810, 0x1814, 0x1818, 0x1860, + ] { + put32(out, at, 1)?; + } + for at in (0x1820..0x1850).step_by(4) { + put32(out, at, u32::MAX)?; + } + Ok(()) +} + +fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { + let p = &i.props; + let period_ms = p[PERIOD_MS]; + if period_ms == 0 { + return Err("the power sample period is 0"); + } + let period_s = f32_div(f32_from_u32(period_ms), f32_from_u32(1000)).ok_or("period")?; + let per_period = + |gain: usize| f32_mul(p[gain], period_s).ok_or("a gain is not a finite normal single"); + let max_scaled = 100 * t.top() as u32; + let base_scaled = 100 * p[BASE_PSTATE]; + let max_power = i.max_power_mw; + let max_power_f = f32_from_u32(max_power); + let mut w = |off: usize, v: u32| put32(out, a(off), v); + w(0x4, p[AIC_CLKS])?; + w(0x8, p[AIC_CLKS])?; + w(0x10, 4)?; + w(0x14, F_1)?; + w(0x2c, 1)?; + w(0x30, 1)?; + w(0x44, base_scaled)?; + w(0x48, 1)?; + w(0x4c, max_scaled)?; + w(0x54, 100)?; + for k in 0..t.len { + w(0x80 + 4 * k, F_1_02)?; + } + w(0x658, 625)?; + // Power controller. + w(0x664, complement(p[PWR_TC])?)?; + w(0x66c, inv(p[PWR_TC])?)?; + w(0x674, p[PWR_KI])?; + w(0x67c, f32_from_u32(p[PWR_MIN_CLAMP]))?; + w(0x680, max_power_f)?; + w(0x684, p[PWR_KP])?; + let k = f32_div(f32_from_u32(max_scaled), max_power_f).ok_or("maximum power is 0")?; + w(0x68c, k ^ 0x8000_0000)?; + w(0x690, p[PWR_MIN_DUTY].wrapping_sub(max_scaled))?; + w(0x698, max_scaled)?; + w(0x6a4, max_power)?; + w(0x6d4, max_scaled)?; + // PPM controller. + let ppm_tc = p[PPM_TC_MS] / period_ms; + w(0x6f0, 4 * ppm_tc)?; + w(0x6fc, complement(ppm_tc)?)?; + w(0x704, inv(ppm_tc)?)?; + w(0x70c, per_period(PPM_KI)?)?; + w(0x718, F_65536)?; + w(0x71c, p[PPM_KP])?; + w(0x728, p[PWR_MIN_DUTY])?; + w(0x72c, max_scaled)?; + w(0x73c, max_power)?; + w(0x744, p[PPM_TC_MS])?; + w(0x74c, ppm_tc * period_ms * BASE_CLOCK_KHZ)?; + // Performance controller. + w(0x788, p[PERF_TGT])?; + w(0x790, p[BOOST_MIN])?; + w(0x794, p[BOOST_CE])?; + w(0x798, p[PERF_RESET])?; + w(0x7a0, 6)?; + w(0x7a4, 1)?; + w(0x7a8, p[PERF_DROP])?; + w(0x7ac, complement(p[PERF_TC])?)?; + w(0x7b0, complement(p[PERF_TC2])?)?; + w(0x7b4, inv(p[PERF_TC])?)?; + w(0x7b8, inv(p[PERF_TC2])?)?; + w(0x7bc, p[PERF_KI])?; + w(0x7c0, p[PERF_KI2])?; + w(0x7c4, f32_from_u32(p[PERF_MIN_CLAMP]))?; + w(0x7c8, F_95)?; + w(0x7cc, p[PERF_KP])?; + w(0x7d0, p[PERF_KP2])?; + let boost = (t.top() as u32) + .checked_sub(p[BASE_PSTATE]) + .ok_or("perf-base-pstate above the top state")?; + w(0x7d4, f32_div(f32_from_u32(boost), F_0_95).ok_or("boost")?)?; + w(0x7d8, base_scaled)?; + w(0x7dc, max_scaled)?; + w(0x7e0, base_scaled)?; + w(0x7ec, p[PERF_TGT])?; + w(0x81c, base_scaled)?; + w(0x838, F_65536)?; + w(0x83c, f32_from_u32(p[PWR_MIN_DUTY]))?; + w(0x840, f32_from_u32(max_scaled))?; + w(0x844, i.main.iter().map(|s| s.mhz).max().unwrap_or(0))?; + w(0x854, p[PWR_MIN_DUTY])?; + w(0x860, F_100)?; + w(0x864, max_scaled)?; + w(0x86c, F_0_8)?; + w(0x870, F_0_2)?; + // Fast-die controller (its sensor mask is a FROM_FILE word). + w(0x8bc, 4)?; + w(0x8d0, F_1)?; + w(0x8d8, per_period(FD_KI)?)?; + w(0x8e4, F_65536)?; + w(0x8e8, p[FD_KP])?; + w(0x8f4, p[PWR_MIN_DUTY])?; + w(0x8f8, max_scaled)?; + w(0x8fc, max_scaled)?; + w(0x900, 100 * p[FD_DELTA])?; + w(0x908, 11000)?; + w(0x938, max_scaled)?; + w(0xbe0, max_scaled)?; + // Power zones and the average-power controller. + w(0xe80, 1)?; + w(0xe84, (i.zones.len() / 3) as u32)?; + for at in [0xe88, 0xe8c, 0xe90] { + w(at, max_power)?; + } + let tgt_tc = p[AVG_TGT_TC]; + w(0xe9c, complement(tgt_tc)?)?; + w(0xea0, inv(tgt_tc)?)?; + w(0xea4, 4 * tgt_tc)?; + w(0xea8, period_ms * tgt_tc)?; + w(0xeac, period_ms * tgt_tc * BASE_CLOCK_KHZ)?; + if i.zones.len() % 3 != 0 || i.zones.len() > 15 { + return Err("the power zones are not up to five u32 triples"); + } + for (z, zone) in i.zones.chunks_exact(3).enumerate() { + let at = 0xeb4 + 0x28 * z; + let (target, offset, tc) = (zone[0], zone[1], zone[2]); + w(at + 0x4, target)?; + w( + at + 0x8, + target + .checked_sub(offset) + .ok_or("a power zone's offset is larger than its target")?, + )?; + w(at + 0xc, 4 * tc)?; + w(at + 0x10, period_ms * tc)?; + w(at + 0x14, 1_320_000_000)?; + w(at + 0x1c, complement(tc)?)?; + w(at + 0x20, inv(tc)?)?; + } + let avg_tc = p[AVG_TC_MS] / period_ms; + w(0xf7c, 4 * avg_tc)?; + w(0xf88, complement(avg_tc)?)?; + w(0xf90, inv(avg_tc)?)?; + w(0xf98, per_period(AVG_KI)?)?; + w(0xfa4, F_65536)?; + w(0xfa8, p[AVG_KP])?; + w(0xfb4, p[AVG_MIN_DUTY])?; + w(0xfb8, max_scaled)?; + w(0xfbc, max_scaled)?; + w(0xfc4, max_power_f)?; + w(0xfc8, max_power)?; + w(0xfd0, p[AVG_TC_MS])?; + w(0xfd8, avg_tc * period_ms * BASE_CLOCK_KHZ)?; + w(0xff8, max_scaled)?; + // The shader-engine controller block at +0x10b4. + const SE: usize = 0x10b4; + let clocks = BASE_CLOCK_KHZ * period_ms; + w(SE + 0x40, 50)?; + w(SE + 0x44, 1)?; + w(SE + 0x4c, complement(p[SE_TC])?)?; + w(SE + 0x50, complement(p[SE_TC1])?)?; + w(SE + 0x54, inv(p[SE_TC])?)?; + w(SE + 0x58, inv(p[SE_TC1])?)?; + w(SE + 0x5c, per_period(SE_KI)?)?; + w(SE + 0x60, per_period(SE_KI1)?)?; + w(SE + 0x68, F_65536)?; + w(SE + 0x6c, p[SE_KP])?; + w(SE + 0x70, p[SE_KP1])?; + w(SE + 0x78, 100)?; + w(SE + 0x7c, max_scaled)?; + w(SE + 0x80, 100)?; + w(SE + 0x88, F_8000)?; + w(SE + 0x8c, p[SE_TARGET])?; + w(SE + 0x94, p[SE_TC] * period_ms)?; + w(SE + 0x98, p[SE_TC1] * period_ms)?; + w(SE + 0x9c, p[SE_TC] * clocks)?; + w(SE + 0xa4, p[SE_TC1] * clocks)?; + w(SE + 0xb0, F_65536)?; + w(SE + 0xd8, 8000)?; + w(SE + 0xdc, 4)?; + w(SE + 0xe0, p[SE_INACT])?; + w(SE + 0xe4, p[SE_CRIT])?; + w(SE + 0xe8, 2)?; + w(SE + 0xec, 4)?; + w(SE + 0xf0, p[SE_RESET])?; + w(SE + 0x11c, F_65536)?; + w(SE + 0x12c, 40)?; + w(SE + 0x130, max_scaled)?; + // Past the second sensor mask (FROM_FILE), the alarm mask (FROM_FILE) and the end flag. + w(0x1290, 125)?; + w(0x1294, 1)?; + w(0x1aa0, 1)?; + w(0x3aa4, F_5)?; + Ok(()) +} + +/// The tail's sensor keys from the six rows (which [`from_file`] fills). +fn tail(out: &mut [u8]) -> R<()> { + let mut g = 0usize; + for row in 0..SENSOR_ROW_COUNT { + let key = get32(out, SENSOR_ROWS + 12 * row)?; + match (key >> 24) as u8 { + b'g' => { + put32(out, SENSOR_G_KEYS + 4 * g, key)?; + g += 1; + } + b'r' => { + put32(out, SENSOR_R_KEY, key)?; + put32(out, SENSOR_R_COUNT, 1)?; + } + b'f' => { + put32(out, SENSOR_F_KEY, key)?; + put32(out, SENSOR_F_COUNT, 1)?; + } + _ => return Err("a sensor row has an unknown kind"), + } + } + if g > 4 { + return Err("more than four g sensor rows"); + } + put32(out, SENSOR_G_MASK, (1u32 << g) - 1)?; + put32(out, HWDATA_A + 0x6d80, 1)?; + put32(out, HWDATA_A + 0x6d84, 1)?; + Ok(()) +} + +/// Copies the [`FROM_FILE`] words of `file` (the board file's record) into `out`. +pub(crate) fn from_file(out: &mut [u8], file: &[u8]) -> R<()> { + for &at in FROM_FILE.iter() { + put32(out, at, get32(file, at)?)?; + } + Ok(()) +} + +/// The first word at which the built record differs from the file's, except the [`UNNEEDED`] +/// words, where the file's value is accepted as long as the built word is zero. +pub(crate) fn cross_check(built: &[u8], file: &[u8]) -> Option<(usize, u32, u32)> { + let word = |s: &[u8], at: usize| -> u32 { + let mut w = [0u8; 4]; + for (i, out) in w.iter_mut().enumerate() { + *out = s.get(at + i).copied().unwrap_or(0); + } + u32::from_le_bytes(w) + }; + (0..built.len().max(file.len())) + .step_by(4) + .map(|at| (at, word(built, at), word(file, at))) + .find(|&(at, b, f)| b != f && !(b == 0 && UNNEEDED.contains(&at))) +} + +/// Builds the record into `out` (zeroed first) from the device-tree inputs; the [`FROM_FILE`] +/// words must have been copied in first ([`from_file`]) or are left as they are. +pub(crate) fn build(out: &mut [u8], i: &Inputs<'_>) -> R<()> { + if out.len() != SIZE { + return Err("record of another size"); + } + let mut keep = [0u32; FROM_FILE.len()]; + for (k, &at) in FROM_FILE.iter().enumerate() { + keep[k] = get32(out, at)?; + } + out.fill(0); + let t = tables(i.main)?; + hwdata_b(out, i, &t)?; + hwdata_a(out, i, &t)?; + // The kept words win over what the rules put there (the relative powers, the masks). + for (k, &at) in FROM_FILE.iter().enumerate() { + put32(out, at, keep[k])?; + } + tail(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn get(b: &[u8], o: usize) -> u32 { + get32(b, o).unwrap() + } + + fn opp(mhz: u32, mv: [u32; 4], mw: u32) -> Opp { + Opp { + mhz, + mv, + cells: 4, + mw, + } + } + + /// Operating points shaped like the J516C's: eight voltages, the upper five with two states. + fn main_opps() -> Vec { + let mut v = vec![opp(0, [125; 4], 0)]; + let groups: [(u32, [u32; 4], &[u32]); 8] = [ + (605, [605, 605, 615, 615], &[338]), + (680, [680, 680, 665, 665], &[618]), + (730, [730, 730, 715, 715], &[796]), + (770, [770, 770, 755, 755], &[832, 924]), + (800, [800, 800, 785, 785], &[952, 1056]), + (860, [855, 860, 845, 845], &[1064, 1182]), + (930, [915, 930, 915, 915], &[1182, 1312]), + (955, [940, 955, 940, 940], &[1242, 1380]), + ]; + for (_, mv, mhz) in groups { + for &m in mhz { + v.push(opp(m, mv, m * 40)); + } + } + v + } + + fn props() -> [u32; PROPS.len()] { + let mut p = [0u32; PROPS.len()]; + p[PERIOD_MS] = 8; + p[BASE_PSTATE] = 1; + p[MIN_SRAM_UV] = 815000; + p[CSAFR_MIN_SRAM_UV] = 815000; + p[AVG_TC_MS] = 32; + p[AVG_KI] = 0x41480000; + p[AVG_KP] = 0x3e99999a; + p[AVG_MIN_DUTY] = 40; + p[AVG_TGT_TC] = 1; + p[FD_KI] = 0x443b8000; + p[FD_KP] = 0x42080000; + p[PERF_TC] = 5; + p[PERF_TC2] = 200; + p[PERF_RESET] = 6; + p[PPM_TC_MS] = 34; + p[PPM_KI] = 0x4198cccd; + p[PPM_KP] = 0x3e99999a; + p[PWR_TC] = 313; + p[PWR_MIN_DUTY] = 40; + p[AIC_CLKS] = 200000; + p[SE_TC] = 9; + p[SE_TC1] = 3; + p[SE_KI] = (-50.0f32).to_bits(); + p[SE_KI1] = (-100.0f32).to_bits(); + p[SE_TARGET] = 1400; + p + } + + fn inputs<'a>(main: &'a [Opp], cs: &'a [Opp], slots: &'a [IoSlot]) -> Inputs<'a> { + Inputs { + props: props(), + zones: &[], + main, + cs, + afr: cs, + slots, + empty_writable: &[4, 6], + ttbat: 0x10bfffb8000, + usc_base: 0x10_0000_0000, + user_end: 0x2ff_ffff_8000, + timestamps: 0xfffffc2ffc000000, + max_power_mw: 54321, + } + } + + #[test] + fn factors() { + assert_eq!(F_1, 1.0f32.to_bits()); + assert_eq!(F_0_2, 0.2f32.to_bits()); + assert_eq!(F_0_8, 0.8f32.to_bits()); + assert_eq!(F_0_95, 0.95f32.to_bits()); + assert_eq!(F_1_02, 1.02f32.to_bits()); + assert_eq!(F_5, 5.0f32.to_bits()); + assert_eq!(F_95, 95.0f32.to_bits()); + assert_eq!(F_100, 100.0f32.to_bits()); + assert_eq!(F_8000, 8000.0f32.to_bits()); + assert_eq!(F_65536, 65536.0f32.to_bits()); + // The filter complement is the correctly rounded (n - 1) / n, also where 1 - 1/n is + // a rounding tie. + assert_eq!(complement(3).unwrap(), (2.0f32 / 3.0).to_bits()); + assert_eq!(complement(313).unwrap(), (312.0f32 / 313.0).to_bits()); + assert_eq!(inv(9).unwrap(), (1.0f32 / 9.0).to_bits()); + assert!(inv(0).is_err()); + } + + #[test] + fn unneeded() { + // No UNNEEDED word is a FROM_FILE word; the cross-check accepts the file's value there + // only against a zero built word, and nothing else. + assert!(UNNEEDED.iter().all(|at| !FROM_FILE.contains(at))); + assert!(UNNEEDED.iter().all(|&at| at % 4 == 0 && at + 4 <= SIZE)); + let mut file = vec![0u8; SIZE]; + for (k, &at) in UNNEEDED.iter().enumerate() { + put32(&mut file, at, 0x2000 + k as u32).unwrap(); + } + let built = vec![0u8; SIZE]; + assert_eq!(cross_check(&built, &file), None); + let mut built = built; + put32(&mut built, UNNEEDED[0], 1).unwrap(); + assert_eq!(cross_check(&built, &file), Some((UNNEEDED[0], 1, 0x2000))); + let mut built = vec![0u8; SIZE]; + put32(&mut built, 0x10, 5).unwrap(); + assert_eq!(cross_check(&built, &file), Some((0x10, 5, 0))); + } + + #[test] + fn layout() { + assert_eq!(a(0x8b8), HWDATA_A + 0x8b8); + assert_eq!(a(0x8bc), HWDATA_A + 0x8c4); + assert_eq!(a(0x938), HWDATA_A + 0x940); + assert_eq!(a(0xbe0), HWDATA_A + 0xde8); + assert_eq!(a(0x1288), HWDATA_A + 0x1490); + assert_eq!(a(0x1290), HWDATA_A + 0x14a0); + assert_eq!(a(0x1a98), HWDATA_A + 0x24a8); + assert_eq!(a(0x1aa0), HWDATA_A + 0x24b8); + assert_eq!(a(0x3aa4), HWDATA_A + 0x64bc); + assert_eq!(MAX_POWER, a(0x6a4)); + // FROM_FILE is sorted, unique and inside the record. + assert!(FROM_FILE.windows(2).all(|w| w[0] < w[1])); + assert!(FROM_FILE.iter().all(|&o| o % 4 == 0 && o + 4 <= SIZE)); + } + + #[test] + fn dvfs_tables() { + let main = main_opps(); + let t = tables(&main).unwrap(); + assert_eq!(t.len, 9); + assert_eq!(&t.primary[..9], &[0, 1, 2, 3, 5, 7, 9, 11, 13]); + assert_eq!(&t.secondary[..9], &[0, 1, 2, 3, 4, 6, 8, 10, 12]); + // One state: nothing to sort. + assert!(tables(&main[..1]).is_err()); + // Frequency falling with voltage. + let mut bad = main.clone(); + bad[2].mhz = 300; + assert!(tables(&bad).is_err()); + // Two states at one voltage with different cluster voltages. + let mut bad = main.clone(); + bad[5].mv[2] = 760; + assert!(tables(&bad).is_err()); + // A cluster whose voltage does not rise. + let mut bad = main.clone(); + bad[13].mv[3] = 915; + assert!(tables(&bad).is_err()); + } + + #[test] + fn record_words() { + let main = main_opps(); + let cs: Vec = [24u32, 480, 624, 810, 1024, 1140, 1236] + .iter() + .zip([635, 635, 680, 720, 790, 855, 875]) + .map(|(&mhz, mv)| Opp { + mhz, + mv: [mv, 0, 0, 0], + cells: 1, + mw: 0, + }) + .collect(); + let slots = [ + IoSlot { + slot: 0, + pa: 0x408d00000, + va: 0xfffffc2068000000, + total: 0x144000, + element: 0x144000, + writable: true, + }, + IoSlot { + slot: 1, + pa: 0x20e101000, + va: 0xfffffc2068149000, + total: 0x4000, + element: 0x4000, + writable: false, + }, + ]; + let i = inputs(&main, &cs, &slots); + let mut r = vec![0u8; SIZE]; + // FROM_FILE words survive the build; the sensor rows feed the tail. + for (k, &at) in FROM_FILE.iter().enumerate() { + put32(&mut r, at, 0x1000 + k as u32).unwrap(); + } + // The file's maximum power is what the input repeats. + put32(&mut r, MAX_POWER, 54321).unwrap(); + let rows = [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"]; + for (k, row) in rows.iter().enumerate() { + put32(&mut r, SENSOR_ROWS + 12 * k, u32::from_le_bytes(**row)).unwrap(); + } + build(&mut r, &i).unwrap(); + assert_eq!(get(&r, 0x1060), 0x1000); + assert_eq!(get(&r, 0x30f0), 0); + // Identity, IOMapping records, flags. + assert_eq!(get(&r, 0xa28), 0x6031); + assert_eq!(get(&r, 0x640), 0x08d00000); + assert_eq!(get(&r, 0x644), 4); + assert_eq!(get(&r, 0x648), 0x68000000); + assert_eq!(get(&r, 0x658), 1); + assert_eq!(get(&r, 0x660), 0x0e101000); + assert_eq!(get(&r, 0x678), 0); + assert_eq!(get(&r, 0x640 + 32 * 4 + 0x18), 1); + assert_eq!(get(&r, 0x640 + 32 * 5 + 0x18), 0); + assert_eq!(get(&r, 0xb44), 0xfffb8000); + assert_eq!(get(&r, 0xb48), 0x10b); + assert_eq!(get(&r, 0xb58), 8); + // Tables: A frequencies, B frequencies, per-cluster lanes, SRAM floor, index maps. + assert_eq!(get(&r, 0xb5c + 4 * 8), 1380); + assert_eq!(get(&r, 0xf9c + 4 * 8), 1242); + assert_eq!(get(&r, 0x1350 + 4 * 4), 832); + assert_eq!(get(&r, 0xb9c + 0x20 + 4 * 2), 615); + assert_eq!(get(&r, 0xb9c + 0x20 + 4 * 5), 605); + assert_eq!(get(&r, 0xd9c + 0x20 + 4 * 2), 815); + assert_eq!(get(&r, 0x1390 + 0x20 * 8 + 4 * 1), 955); + assert_eq!(get(&r, 0x10dc + 4 * 8), 13); + assert_eq!(get(&r, 0x111c + 4 * 8), 12); + assert_eq!(get(&r, 0x107c), 100); + assert_eq!(get(&r, 0x109c + 4 * 8), 100); + // CS table: top twice, then the arrays with the SRAM floor. + assert_eq!(get(&r, 0x115c), 6); + assert_eq!(get(&r, 0x1160), 6); + assert_eq!(get(&r, 0x1164), 24); + assert_eq!(get(&r, 0x11a4), 635); + assert_eq!(get(&r, 0x1224), 815); + assert_eq!(get(&r, 0x12a4), 6); + assert_eq!(get(&r, 0x12ac), 24); + // HardwareDataA: scaled states, the maximum power and its derivatives, filters, SE. + assert_eq!(get(&r, a(0x4c)), 800); + assert_eq!(get(&r, a(0x680)), 54321.0f32.to_bits()); + assert_eq!(get(&r, a(0x68c)), (-800.0f32 / 54321.0).to_bits()); + assert_eq!(get(&r, a(0x690)), (40i32 - 800) as u32); + assert_eq!(get(&r, MAX_POWER), 54321); + assert_eq!(get(&r, a(0x6fc)), (3.0f32 / 4.0).to_bits()); + assert_eq!(get(&r, a(0x74c)), 4 * 8 * 24000); + assert_eq!(get(&r, a(0x7d4)), (7.0f32 / 0.95).to_bits()); + assert_eq!(get(&r, a(0x844)), 1380); + assert_eq!(get(&r, a(0x8d8)), 0x40c00001); + assert_eq!(get(&r, a(0x8e8)), 0x42080000); + assert_eq!(get(&r, a(0xe88)), 54321); + assert_eq!(get(&r, a(0xf88)), 0x3f400000); + assert_eq!(get(&r, a(0xfd8)), 4 * 8 * 24000); + assert_eq!(get(&r, a(0x10b4 + 0x4c)), (8.0f32 / 9.0).to_bits()); + assert_eq!(get(&r, a(0x10b4 + 0x5c)), (-50.0f32 * 0.008).to_bits()); + assert_eq!(get(&r, a(0x10b4 + 0x9c)), 9 * 192000); + assert_eq!(get(&r, a(0x3aa4)), F_5); + // Tail keys. + assert_eq!(get(&r, SENSOR_G_KEYS), u32::from_le_bytes(*b"RT0g")); + assert_eq!(get(&r, SENSOR_G_KEYS + 12), u32::from_le_bytes(*b"RT3g")); + assert_eq!(get(&r, SENSOR_G_MASK), 0xf); + assert_eq!(get(&r, SENSOR_R_KEY), u32::from_le_bytes(*b"RT0r")); + assert_eq!(get(&r, SENSOR_F_COUNT), 1); + // Refusals. + let mut short = vec![0u8; SIZE - 4]; + assert!(build(&mut short, &i).is_err()); + let mut j = inputs(&main, &cs, &slots); + j.props[PERIOD_MS] = 0; + assert!(build(&mut r, &j).is_err()); + let mut j = inputs(&main, &cs, &slots); + j.empty_writable = &[0]; + assert!(build(&mut r, &j).is_err()); + let mut j = inputs(&main, &cs, &slots); + j.max_power_mw = 0; + assert!(build(&mut r, &j).is_err()); + let mut j = inputs(&main, &cs, &slots); + j.zones = &[100, 101, 1]; + assert!(build(&mut r, &j).is_err()); + } + + // --- The fixture: a saved copy of a J516C's device tree and its private board file. --- + + fn be32(b: &[u8], o: usize) -> u32 { + u32::from_be_bytes(b[o..o + 4].try_into().unwrap()) + } + + /// All nodes of a flattened device tree: path -> properties. + fn fdt(dtb: &[u8]) -> Vec<(String, Vec<(String, Vec)>)> { + assert_eq!(be32(dtb, 0), 0xd00dfeed); + let strings = be32(dtb, 12) as usize; + let mut at = be32(dtb, 8) as usize; + let mut path: Vec = Vec::new(); + let mut out: Vec<(String, Vec<(String, Vec)>)> = Vec::new(); + loop { + let token = be32(dtb, at); + at += 4; + match token { + 1 => { + let end = dtb[at..].iter().position(|&b| b == 0).unwrap(); + path.push(String::from_utf8(dtb[at..at + end].to_vec()).unwrap()); + at = (at + end + 1).next_multiple_of(4); + out.push((path.join("/"), Vec::new())); + } + 2 => { + path.pop(); + } + 3 => { + let len = be32(dtb, at) as usize; + let name_off = strings + be32(dtb, at + 4) as usize; + let name_end = dtb[name_off..].iter().position(|&b| b == 0).unwrap(); + let name = + String::from_utf8(dtb[name_off..name_off + name_end].to_vec()).unwrap(); + let value = dtb[at + 8..at + 8 + len].to_vec(); + at = (at + 8 + len).next_multiple_of(4); + let node = path.join("/"); + out.iter_mut() + .rev() + .find(|(p, _)| *p == node) + .unwrap() + .1 + .push((name, value)); + } + 4 => {} + 9 => return out, + _ => panic!("fdt token {token}"), + } + } + } + + fn prop<'a>(props: &'a [(String, Vec)], name: &str) -> Option<&'a [u8]> { + props + .iter() + .find(|(n, _)| n == name) + .map(|(_, v)| v.as_slice()) + } + + fn opp_table( + nodes: &[(String, Vec<(String, Vec)>)], + phandle: u32, + cells: usize, + ) -> Vec { + let (table, _) = nodes + .iter() + .find(|(_, p)| prop(p, "phandle").map(|v| be32(v, 0)) == Some(phandle)) + .unwrap(); + let mut rows: Vec<(String, Opp)> = nodes + .iter() + .filter(|(p, _)| { + p.starts_with(&format!("{table}/")) + && p.matches('/').count() == table.matches('/').count() + 1 + }) + .map(|(p, props)| { + let hz = u64::from_be_bytes(prop(props, "opp-hz").unwrap().try_into().unwrap()); + let uv = prop(props, "opp-microvolt").unwrap(); + assert_eq!(uv.len(), 4 * cells); + let mut mv = [0u32; CLUSTERS]; + for (c, m) in mv.iter_mut().enumerate().take(cells) { + *m = be32(uv, 4 * c).div_ceil(1000); + } + let mw = prop(props, "opp-microwatt").map_or(0, |w| be32(w, 0) / 1000); + ( + p.clone(), + Opp { + mhz: (hz / 1_000_000) as u32, + mv, + cells, + mw, + }, + ) + }) + .collect(); + rows.sort_by(|x, y| x.0.cmp(&y.0)); + rows.into_iter().map(|(_, o)| o).collect() + } + + /// Builder on the saved device tree + the file's FROM_FILE words == the file's hardware + /// record, byte for byte. Needs `G15_J516C_FIXTURES=` with `j516c.dtb` and + /// `t6031-j516c-board.bin` (off the tree): ignored by default, `--ignored` runs it. + #[test] + #[ignore = "needs the off-tree J516C fixtures (G15_J516C_FIXTURES)"] + fn j516c_fixture() { + let dir = std::env::var_os("G15_J516C_FIXTURES") + .expect("G15_J516C_FIXTURES names the directory with j516c.dtb and the board file"); + let dir = std::path::Path::new(&dir); + let dtb = std::fs::read(dir.join("j516c.dtb")).unwrap(); + let blob = std::fs::read(dir.join("t6031-j516c-board.bin")).unwrap(); + assert_eq!(&blob[..8], b"T6031BRD"); + let at = 16 + 3 * 8; + let file = &blob[at..at + SIZE]; + + let nodes = fdt(&dtb); + let (_, gpu) = nodes + .iter() + .find(|(_, p)| { + prop(p, "compatible") + .is_some_and(|c| c.split(|&b| b == 0).any(|s| s == b"apple,agx-t6031")) + }) + .expect("a T6031 GPU node"); + let mut props = [0u32; PROPS.len()]; + for (v, (name, default)) in props.iter_mut().zip(PROPS.iter()) { + *v = match prop(gpu, name.to_str().unwrap()) { + Some(b) => be32(b, 0), + None => default.unwrap_or_else(|| panic!("GPU node has no {name:?}")), + }; + } + let zones: Vec = prop(gpu, "apple,power-zones") + .map(|b| b.chunks(4).map(|c| be32(c, 0)).collect()) + .unwrap_or_default(); + let main = opp_table( + &nodes, + be32(prop(gpu, "operating-points-v2").unwrap(), 0), + 4, + ); + let cs = opp_table(&nodes, be32(prop(gpu, "apple,cs-opp").unwrap(), 0), 1); + let afr = opp_table(&nodes, be32(prop(gpu, "apple,afr-opp").unwrap(), 0), 1); + assert_eq!(main.len(), PERF_STATES); + // The ttbs region: the first memory-region phandle's reserved-memory node. + let regions = prop(gpu, "memory-region").unwrap(); + let (_, ttbs) = nodes + .iter() + .find(|(_, p)| prop(p, "phandle").map(|v| be32(v, 0)) == Some(be32(regions, 0))) + .unwrap(); + let reg = prop(ttbs, "reg").unwrap(); + let ttbat = u64::from_be_bytes(reg[..8].try_into().unwrap()); + // The IOMapping slots: the T6031 table's addresses and sizes with the file's VAs (the + // VAs are the driver's; the layout packs them from a base with a page gap). + let slots_table: [(u8, u64, u32, u32, bool); 17] = [ + (0, 0x408d00000, 0x144000, 0x144000, true), + (1, 0x20e100000, 0x4000, 0x4000, false), + (2, 0x292414000, 0x4000, 0x4000, true), + (3, 0x408000000, 0x20000, 0x20000, true), + (7, 0x290444000, 0x4000, 0x4000, false), + (9, 0x408e08000, 0x8000, 0x8000, true), + (11, 0x220000000, 8 * 0xd8000, 0xd8000, true), + (12, 0x29244c000, 0x4000, 0x4000, false), + (18, 0x2903d0000, 0x4000, 0x4000, true), + (19, 0x2903c0000, 0x4000, 0x4000, false), + (20, 0x2903d8000, 0x4000, 0x4000, true), + (21, 0x408eb4000, 0x4000, 0x4000, true), + (24, 0x409000000, 0x8000, 0x8000, true), + (25, 0x3c945c000, 0x4000, 0x4000, true), + (26, 0x290400000, 0x8000, 0x8000, false), + (28, 0x2a4dc0000, 0x24000, 0x24000, false), + (29, 0x408e1c000, 0x4000, 0x4000, false), + ]; + let mut va = 0xfffffc2068000000u64; + let slots: Vec = slots_table + .iter() + .map(|&(slot, pa, total, element, writable)| { + let delta = pa & 0x3fff; + let s = IoSlot { + slot, + pa, + va: va + delta, + total, + element, + writable, + }; + va += (delta + u64::from(total) + 0x3fff) & !0x3fff; + va += 0x4000; + s + }) + .collect(); + let i = Inputs { + props, + zones: &zones, + main: &main, + cs: &cs, + afr: &afr, + slots: &slots, + empty_writable: &[4, 6, 8, 10, 13, 14, 15, 16, 17], + ttbat, + usc_base: 0x10_0000_0000, + user_end: 0x2ff_ffff_8000, + timestamps: 0xfffffc2ffc000000, + max_power_mw: get(file, MAX_POWER), + }; + let mut built = vec![0u8; SIZE]; + from_file(&mut built, file).unwrap(); + build(&mut built, &i).unwrap(); + assert_eq!(cross_check(&built, file), None, "(offset, built, file)"); + for &at in UNNEEDED.iter() { + assert_eq!(get(&built, at), 0); + } + } +} diff --git a/drivers/gpu/drm/asahi/g15/board/template.rs b/drivers/gpu/drm/asahi/g15/board/template.rs new file mode 100644 index 0000000000000..3a53939ae08c0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/board/template.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Sparse board-image templates: 16-byte lines over a zero image. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. +//! The hash check of an expanded template is the kernel side's ([`super::expand`]). + +/// One template line: 16 bytes at `offset`. +pub(crate) struct Line { + pub(crate) offset: u32, + pub(crate) data: [u8; 16], +} + +/// A sparse template. +pub(crate) struct Template { + pub(crate) size: usize, + pub(crate) lines: &'static [Line], + pub(crate) sha256: [u8; 32], +} + +/// Fills the first `template.size` bytes of `out` with the template: zero, then its lines. `Err` +/// when `out` is too short or a line starts past the end. +pub(crate) fn fill(template: &Template, out: &mut [u8]) -> Result<(), ()> { + let out = out.get_mut(..template.size).ok_or(())?; + out.fill(0); + for line in template.lines { + let offset = line.offset as usize; + if offset >= template.size { + return Err(()); + } + let n = 16.min(template.size - offset); + out[offset..offset + n].copy_from_slice(&line.data[..n]); + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g15/board/words.rs b/drivers/gpu/drm/asahi/g15/board/words.rs new file mode 100644 index 0000000000000..4c5082465d428 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/board/words.rs @@ -0,0 +1,653 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The J516C Globals and Power images from the device tree. +//! +//! The 2419 firmware's Globals (15,868 bytes) and Power (2,292 bytes) are, on the J516C, the G15S +//! templates ([`super::data`]) with these words of its own: +//! +//! - The GPU node's power words ([`PROPS`]): `apple,idleoff-standby-timer` (Globals +0x7a8); +//! in Power the average-power controller (+0x10 filter periods = `apple,avg-power-filter-tc-ms` +//! / `apple,power-sample-period`, +0x14 `apple,avg-power-ki-only` x period, +0x18 +//! `apple,avg-power-kp`, +0x1c `apple,avg-power-min-duty-cycle`, +0x20 +//! `apple,avg-power-target-filter-tc`), the power zones (+0xc count, +0x24 up to five +//! {target, target - offset, filter} rows of `apple,power-zones`), the fast-die controller +//! (+0xb8 `apple,fast-die0-proportional-gain`, +0xbc `apple,fast-die0-integral-gain` x period) +//! and the PPM controller (+0xd4 `apple,ppm-kp`, +0xd8 `apple,ppm-ki` x period). A gain x +//! period is the IEEE single product of the gain and `period_ms / 1000`, each rounded to +//! single ([`f32_mul`], [`f32_div`]). +//! - The maximum power: the hardware record's HardwareDataA +0x6a4 word (Globals +0x78, Power +//! +0xd0). The G15S template carries its own machine's there too. +//! - The performance-state cap: 100 x the top firmware DVFS state, 8 (Globals +0x7c, +0x80, +//! +0x8c). +//! - Words the device tree does not carry: command submission enabled at initdata (Globals +//! +0x70 = 1; the G15S template has 0), the start path's +0x980 = 100 (written again at +//! start on every SoC), the soft-fault word (u32 3 at the unaligned +0xdf9 of the G15S image) +//! at +0x1201, where the G15C Globals has it, and the Power +0x244 flag = 1. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. + +use core::ffi::CStr; + +pub(crate) const GLOBALS_SIZE: usize = 15868; +pub(crate) const POWER_SIZE: usize = 2292; + +/// HardwareDataA in the hardware record (= `fw::initdata::hwdata::HWDATA_A`) and its maximum +/// power word (= `fw::initdata::hwdata::MAX_POWER_A`). +pub(crate) const HWDATA_A: usize = 0x4580; +pub(crate) const HWDATA_A_MAX_POWER: usize = 0x6a4; + +/// The top firmware DVFS state (= `board::DVFS_STATES - 1`). +pub(crate) const TOP_STATE: u32 = 8; + +/// Globals words. +mod globals { + pub(super) const COMMAND_SUBMISSION: usize = 0x70; + pub(super) const MAX_POWER: usize = 0x78; + /// The power-interface targets 1 and 2 and the performance-state cap. + pub(super) const STATE_CAP: [usize; 3] = [0x7c, 0x80, 0x8c]; + pub(super) const STANDBY_TIMER: usize = 0x7a8; + pub(super) const UNK_980: usize = 0x980; + /// The soft-fault word: unaligned, in the G15S and the G15C image. + pub(super) const SOFT_FAULT_G15S: usize = 0xdf9; + pub(super) const SOFT_FAULT_G15C: usize = 0x1201; + pub(super) const SOFT_FAULT: u32 = 3; +} + +/// Power words. +mod power { + pub(super) const ZONE_COUNT: usize = 0x0c; + pub(super) const AVG_FILTER_PERIODS: usize = 0x10; + pub(super) const AVG_KI_DT: usize = 0x14; + pub(super) const AVG_KP: usize = 0x18; + pub(super) const AVG_MIN_DUTY_CYCLE: usize = 0x1c; + pub(super) const AVG_TARGET_FILTER_TC: usize = 0x20; + pub(super) const ZONES: usize = 0x24; + pub(super) const ZONE_STRIDE: usize = 12; + pub(super) const FAST_DIE_KP: usize = 0xb8; + pub(super) const FAST_DIE_KI_DT: usize = 0xbc; + pub(super) const MAX_POWER: usize = 0xd0; + pub(super) const PPM_KP: usize = 0xd4; + pub(super) const PPM_KI_DT: usize = 0xd8; + pub(super) const FLAG_244: usize = 0x244; +} + +/// The GPU node's power properties, each one u32 (gains are IEEE single bits), in the order of +/// [`PowerWords::values`]. +pub(crate) const PROPS: [&CStr; 11] = [ + c"apple,power-sample-period", + c"apple,idleoff-standby-timer", + c"apple,avg-power-filter-tc-ms", + c"apple,avg-power-ki-only", + c"apple,avg-power-kp", + c"apple,avg-power-min-duty-cycle", + c"apple,avg-power-target-filter-tc", + c"apple,fast-die0-proportional-gain", + c"apple,fast-die0-integral-gain", + c"apple,ppm-kp", + c"apple,ppm-ki", +]; +pub(crate) const PERIOD_MS: usize = 0; +pub(crate) const STANDBY_TIMER: usize = 1; +pub(crate) const AVG_POWER_FILTER_TC_MS: usize = 2; +pub(crate) const AVG_POWER_KI_ONLY: usize = 3; +pub(crate) const AVG_POWER_KP: usize = 4; +pub(crate) const AVG_POWER_MIN_DUTY_CYCLE: usize = 5; +pub(crate) const AVG_POWER_TARGET_FILTER_TC: usize = 6; +pub(crate) const FAST_DIE0_PROPORTIONAL_GAIN: usize = 7; +pub(crate) const FAST_DIE0_INTEGRAL_GAIN: usize = 8; +pub(crate) const PPM_KP: usize = 9; +pub(crate) const PPM_KI: usize = 10; + +/// The power zones, u32 triples {target, offset, filter time constant}; at most five. +pub(crate) const ZONES: &CStr = c"apple,power-zones"; +pub(crate) const MAX_ZONES: usize = 5; + +/// The GPU node's power words. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PowerWords { + pub(crate) values: [u32; PROPS.len()], + pub(crate) zones: [[u32; 3]; MAX_ZONES], + pub(crate) zone_count: u32, +} + +impl PowerWords { + /// `values` in [`PROPS`] order and the `apple,power-zones` cells. + pub(crate) fn new(values: [u32; PROPS.len()], zones: &[u32]) -> Result { + if values[PERIOD_MS] == 0 { + return Err("the power sample period is 0"); + } + if zones.len() % 3 != 0 || zones.len() > 3 * MAX_ZONES { + return Err("the power zones are not up to five u32 triples"); + } + let mut out = Self { + values, + zones: [[0; 3]; MAX_ZONES], + zone_count: (zones.len() / 3) as u32, + }; + for (row, cells) in out.zones.iter_mut().zip(zones.chunks_exact(3)) { + row.copy_from_slice(cells); + } + Ok(out) + } + + /// The sample period in seconds, IEEE single bits. + fn period_s(&self) -> Result { + f32_div(f32_from_u32(self.values[PERIOD_MS]), f32_from_u32(1000)) + .ok_or("the power sample period has no single-precision quotient") + } + + /// A gain x the sample period, IEEE single bits. + fn per_period(&self, gain: usize) -> Result { + f32_mul(self.values[gain], self.period_s()?) + .ok_or("a gain is not a finite normal single, or its product with the period is not") + } +} + +fn get32(b: &[u8], offset: usize) -> Result { + let w = b.get(offset..offset + 4).ok_or("image too short")?; + Ok(u32::from_le_bytes([w[0], w[1], w[2], w[3]])) +} + +fn put32(b: &mut [u8], offset: usize, value: u32) -> Result<(), &'static str> { + b.get_mut(offset..offset + 4) + .ok_or("image too short")? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) +} + +/// Writes the J516C words into `image`, the expanded G15S Globals template. +pub(crate) fn globals( + image: &mut [u8], + max_power_mw: u32, + w: &PowerWords, +) -> Result<(), &'static str> { + if image.len() != GLOBALS_SIZE { + return Err("Globals image of another size"); + } + put32(image, globals::COMMAND_SUBMISSION, 1)?; + put32(image, globals::MAX_POWER, max_power_mw)?; + for at in globals::STATE_CAP { + put32(image, at, 100 * TOP_STATE)?; + } + put32(image, globals::STANDBY_TIMER, w.values[STANDBY_TIMER])?; + put32(image, globals::UNK_980, 100)?; + if get32(image, globals::SOFT_FAULT_G15S)? != globals::SOFT_FAULT { + return Err("Globals template without the soft-fault word at its G15S place"); + } + put32(image, globals::SOFT_FAULT_G15S, 0)?; + put32(image, globals::SOFT_FAULT_G15C, globals::SOFT_FAULT)?; + Ok(()) +} + +/// Writes the J516C words into `image`, the expanded G15S Power template. +pub(crate) fn power( + image: &mut [u8], + max_power_mw: u32, + w: &PowerWords, +) -> Result<(), &'static str> { + if image.len() != POWER_SIZE { + return Err("Power image of another size"); + } + let periods = w.values[AVG_POWER_FILTER_TC_MS] / w.values[PERIOD_MS]; + if periods == 0 { + return Err("the average-power filter time constant is shorter than the sample period"); + } + put32(image, power::ZONE_COUNT, w.zone_count)?; + put32(image, power::AVG_FILTER_PERIODS, periods)?; + put32(image, power::AVG_KI_DT, w.per_period(AVG_POWER_KI_ONLY)?)?; + put32(image, power::AVG_KP, w.values[AVG_POWER_KP])?; + put32( + image, + power::AVG_MIN_DUTY_CYCLE, + w.values[AVG_POWER_MIN_DUTY_CYCLE], + )?; + put32( + image, + power::AVG_TARGET_FILTER_TC, + w.values[AVG_POWER_TARGET_FILTER_TC], + )?; + for (i, zone) in w.zones.iter().enumerate() { + let at = power::ZONES + power::ZONE_STRIDE * i; + let (target, offset, filter) = (zone[0], zone[1], zone[2]); + let target_off = target + .checked_sub(offset) + .ok_or("a power zone's offset is larger than its target")?; + put32(image, at, target)?; + put32(image, at + 4, target_off)?; + put32(image, at + 8, filter)?; + } + put32( + image, + power::FAST_DIE_KP, + w.values[FAST_DIE0_PROPORTIONAL_GAIN], + )?; + put32( + image, + power::FAST_DIE_KI_DT, + w.per_period(FAST_DIE0_INTEGRAL_GAIN)?, + )?; + put32(image, power::MAX_POWER, max_power_mw)?; + put32(image, power::PPM_KP, w.values[PPM_KP])?; + put32(image, power::PPM_KI_DT, w.per_period(PPM_KI)?)?; + put32(image, power::FLAG_244, 1)?; + Ok(()) +} + +/// The first word (4-byte aligned offset) at which `a` and `b` differ, with both values; a +/// missing word reads 0. `None` when they are equal. +pub(crate) fn first_difference(a: &[u8], b: &[u8]) -> Option<(usize, u32, u32)> { + let word = |s: &[u8], at: usize| -> u32 { + let mut w = [0u8; 4]; + for (i, out) in w.iter_mut().enumerate() { + *out = s.get(at + i).copied().unwrap_or(0); + } + u32::from_le_bytes(w) + }; + let len = a.len().max(b.len()); + (0..len) + .step_by(4) + .map(|at| (at, word(a, at), word(b, at))) + .find(|&(_, x, y)| x != y) +} + +// --- IEEE binary32 arithmetic on bit patterns (the kernel has no floating point) --- +// +// Operands and results are finite normal numbers or zero; anything else is `None`. +// Rounding is to nearest, ties to even. + +/// Sign, unbiased exponent of the leading bit and mantissa with its implicit bit; zero is +/// `(sign, 0, 0)`. +fn unpack(bits: u32) -> Option<(bool, i32, u32)> { + let sign = bits >> 31 != 0; + let exponent = ((bits >> 23) & 0xff) as i32; + let mantissa = bits & 0x7f_ffff; + match exponent { + 0 if mantissa == 0 => Some((sign, 0, 0)), + 0 | 0xff => None, + _ => Some((sign, exponent - 127, mantissa | 0x80_0000)), + } +} + +/// The single nearest to `sign * m * 2^exp`, `m` with its leading bit anywhere; a sticky bit in +/// the lowest bit of `m` may stand for a nonzero remainder below it. +fn pack(sign: bool, exp: i32, m: u64) -> Option { + if m == 0 { + return Some((sign as u32) << 31); + } + let shift = m.leading_zeros(); + let m = m << shift; + let exp = exp - shift as i32 + 63; + let mut mantissa = (m >> 40) as u32; + let rest = m & ((1u64 << 40) - 1); + let half = 1u64 << 39; + if rest > half || (rest == half && mantissa & 1 == 1) { + mantissa += 1; + } + let mut exp = exp; + if mantissa == 1 << 24 { + mantissa >>= 1; + exp += 1; + } + let biased = exp + 127; + if !(1..=254).contains(&biased) { + return None; + } + Some((sign as u32) << 31 | (biased as u32) << 23 | (mantissa & 0x7f_ffff)) +} + +/// `n` as a single. +pub(crate) fn f32_from_u32(n: u32) -> u32 { + // Every u32 rounds to a normal single or zero. + pack(false, 0, n as u64).unwrap_or(0) +} + +/// `a * b`. +pub(crate) fn f32_mul(a: u32, b: u32) -> Option { + let (sa, ea, ma) = unpack(a)?; + let (sb, eb, mb) = unpack(b)?; + if ma == 0 || mb == 0 { + return Some(((sa ^ sb) as u32) << 31); + } + pack(sa ^ sb, ea + eb - 46, ma as u64 * mb as u64) +} + +/// `a / b`. +pub(crate) fn f32_div(a: u32, b: u32) -> Option { + let (sa, ea, ma) = unpack(a)?; + let (sb, eb, mb) = unpack(b)?; + if mb == 0 { + return None; + } + if ma == 0 { + return Some(((sa ^ sb) as u32) << 31); + } + // ma < 2^24, so ma << 40 fits; the quotient has at least 40 bits, 16 and more below the + // rounding position, so a sticky lowest bit stands in for a nonzero remainder. + let n = (ma as u64) << 40; + let mut q = n / mb as u64; + if n % mb as u64 != 0 { + q |= 1; + } + pack(sa ^ sb, ea - eb - 40, q) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn get32(b: &[u8], o: usize) -> u32 { + u32::from_le_bytes(b[o..o + 4].try_into().unwrap()) + } + + /// The J516C GPU node's power words, as the saved device tree of that machine carries them. + fn j516c_words() -> PowerWords { + PowerWords::new( + [ + 8, // apple,power-sample-period + 1500, // apple,idleoff-standby-timer + 32, // apple,avg-power-filter-tc-ms + 0x41480000, // apple,avg-power-ki-only (12.5) + 0x3e99999a, // apple,avg-power-kp (0.3) + 40, // apple,avg-power-min-duty-cycle + 1, // apple,avg-power-target-filter-tc + 0x42080000, // apple,fast-die0-proportional-gain (34.0) + 0x443b8000, // apple,fast-die0-integral-gain (750.0) + 0x3e99999a, // apple,ppm-kp (0.3) + 0x4198cccd, // apple,ppm-ki (19.1) + ], + &[], + ) + .unwrap() + } + + #[test] + fn single_arithmetic() { + // The products the power words need, against the host's IEEE singles. + let dt = f32_div(f32_from_u32(8), f32_from_u32(1000)).unwrap(); + assert_eq!(dt, (8.0f32 / 1000.0).to_bits()); + for ki in [0x443b8000u32, 0x41480000, 0x4198cccd] { + assert_eq!( + f32_mul(ki, dt).unwrap(), + (f32::from_bits(ki) * f32::from_bits(dt)).to_bits() + ); + } + // A sweep of normal operands, both signs. + let mut x = 0x2545f491u32; + for _ in 0..20000 { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + let a = f32::from_bits(x); + let b = f32::from_bits(x.rotate_left(11) ^ 0x5a5a5a5a); + if !a.is_normal() || !b.is_normal() { + continue; + } + let p = a * b; + if p.is_normal() { + assert_eq!( + f32_mul(a.to_bits(), b.to_bits()), + Some(p.to_bits()), + "{a} * {b}" + ); + } + let q = a / b; + if q.is_normal() { + assert_eq!( + f32_div(a.to_bits(), b.to_bits()), + Some(q.to_bits()), + "{a} / {b}" + ); + } + } + for n in [ + 0u32, + 1, + 8, + 1000, + 1500, + 0xffffff, + 0x1000000, + 0x1000001, + u32::MAX, + ] { + assert_eq!(f32_from_u32(n), (n as f32).to_bits()); + } + assert_eq!(f32_mul(0, 0x3f800000), Some(0)); + assert_eq!(f32_mul(0x3f800000, 0x80000000), Some(0x80000000)); + assert_eq!(f32_div(0x3f800000, 0), None); + assert_eq!(f32_div(0, 0x3f800000), Some(0)); + // Infinities, NaNs and subnormals are not operands. + assert_eq!(f32_mul(0x7f800000, 0x3f800000), None); + assert_eq!(f32_mul(0x00000001, 0x3f800000), None); + // Overflow and underflow of the result are refused, not rounded. + assert_eq!(f32_mul(0x7f000000, 0x7f000000), None); + assert_eq!(f32_mul(0x00800000, 0x00800000), None); + } + + #[test] + fn words_shape() { + let w = j516c_words(); + assert_eq!(w.zone_count, 0); + assert_eq!(PowerWords::new(w.values, &[1, 2, 3]).unwrap().zone_count, 1); + assert_eq!( + PowerWords::new(w.values, &[1, 2, 3]).unwrap().zones[0], + [1, 2, 3] + ); + assert!(PowerWords::new(w.values, &[1, 2]).is_err()); + assert!(PowerWords::new(w.values, &[0; 18]).is_err()); + let mut v = w.values; + v[PERIOD_MS] = 0; + assert!(PowerWords::new(v, &[]).is_err()); + assert_eq!(PROPS.len(), w.values.len()); + assert_eq!(PROPS[PERIOD_MS], c"apple,power-sample-period"); + assert_eq!(PROPS[STANDBY_TIMER], c"apple,idleoff-standby-timer"); + } + + #[test] + fn globals_words() { + let w = j516c_words(); + let mut img = vec![0u8; GLOBALS_SIZE]; + // A G15S-shaped image: the soft-fault word at its unaligned place. + img[0xdf9] = 3; + globals(&mut img, 54321, &w).unwrap(); + assert_eq!(get32(&img, 0x70), 1); + assert_eq!(get32(&img, 0x78), 54321); + assert_eq!(get32(&img, 0x7c), 800); + assert_eq!(get32(&img, 0x80), 800); + assert_eq!(get32(&img, 0x8c), 800); + assert_eq!(get32(&img, 0x7a8), 1500); + assert_eq!(get32(&img, 0x980), 100); + assert_eq!(get32(&img, 0xdf8), 0); + assert_eq!(get32(&img, 0x1200), 0x300); + // Everything else untouched. + let touched = [0x70, 0x78, 0x7c, 0x80, 0x8c, 0x7a8, 0x980, 0xdf8, 0x1200]; + for o in (0..GLOBALS_SIZE).step_by(4) { + if !touched.contains(&o) { + assert_eq!(get32(&img, o), 0, "+{o:#x}"); + } + } + // Without the soft-fault word where the G15S image has it, the image is not the one + // this rule is written for. + let mut bad = vec![0u8; GLOBALS_SIZE]; + assert!(globals(&mut bad, 1, &w).is_err()); + let mut short = vec![0u8; GLOBALS_SIZE - 1]; + assert!(globals(&mut short, 1, &w).is_err()); + } + + #[test] + fn power_words() { + let w = j516c_words(); + let mut img = vec![0u8; POWER_SIZE]; + power(&mut img, 54321, &w).unwrap(); + assert_eq!(get32(&img, 0x0c), 0); + assert_eq!(get32(&img, 0x10), 4); + assert_eq!(get32(&img, 0x14), 0x3dcccccd); + assert_eq!(get32(&img, 0x18), 0x3e99999a); + assert_eq!(get32(&img, 0x1c), 40); + assert_eq!(get32(&img, 0x20), 1); + assert_eq!(get32(&img, 0xb8), 0x42080000); + assert_eq!(get32(&img, 0xbc), 0x40c00001); + assert_eq!(get32(&img, 0xd0), 54321); + assert_eq!(get32(&img, 0xd4), 0x3e99999a); + assert_eq!(get32(&img, 0xd8), 0x3e1c779b); + assert_eq!(get32(&img, 0x244), 1); + let touched = [ + 0x10, 0x14, 0x18, 0x1c, 0x20, 0xb8, 0xbc, 0xd0, 0xd4, 0xd8, 0x244, + ]; + for o in (0..POWER_SIZE).step_by(4) { + if !touched.contains(&o) { + assert_eq!(get32(&img, o), 0, "+{o:#x}"); + } + } + // Power zones: count and {target, target - offset, filter} rows. + let z = PowerWords::new(w.values, &[5000, 200, 7, 9000, 1000, 3]).unwrap(); + let mut img = vec![0u8; POWER_SIZE]; + power(&mut img, 1, &z).unwrap(); + assert_eq!(get32(&img, 0x0c), 2); + assert_eq!(get32(&img, 0x24), 5000); + assert_eq!(get32(&img, 0x28), 4800); + assert_eq!(get32(&img, 0x2c), 7); + assert_eq!(get32(&img, 0x30), 9000); + assert_eq!(get32(&img, 0x34), 8000); + assert_eq!(get32(&img, 0x38), 3); + assert_eq!(get32(&img, 0x3c), 0); + // An offset larger than its target has no target - offset word. + let under = PowerWords::new(w.values, &[100, 101, 1]).unwrap(); + let mut img = vec![0u8; POWER_SIZE]; + assert!(power(&mut img, 1, &under).is_err()); + // A filter time constant shorter than the sample period has no period count. + let mut v = w.values; + v[AVG_POWER_FILTER_TC_MS] = 4; + let mut img = vec![0u8; POWER_SIZE]; + assert!(power(&mut img, 1, &PowerWords::new(v, &[]).unwrap()).is_err()); + let mut short = vec![0u8; POWER_SIZE - 1]; + assert!(power(&mut short, 1, &w).is_err()); + } + + #[test] + fn differences() { + let a = [1u8, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0]; + let mut b = a; + assert_eq!(first_difference(&a, &b), None); + b[9] = 1; + assert_eq!(first_difference(&a, &b), Some((8, 3, 0x103))); + assert_eq!(first_difference(&a, &b[..8]), Some((8, 3, 0))); + assert_eq!(first_difference(&a[..8], &b), Some((8, 0, 0x103))); + } + + // --- The fixture: a saved copy of a J516C's device tree and its private board file. --- + + fn be32(b: &[u8], o: usize) -> u32 { + u32::from_be_bytes(b[o..o + 4].try_into().unwrap()) + } + + /// Properties of the flattened device tree's node whose `compatible` names `compatible`. + fn fdt_node_props(dtb: &[u8], compatible: &[u8]) -> Option)>> { + assert_eq!(be32(dtb, 0), 0xd00dfeed); + let strings = be32(dtb, 12) as usize; + let mut at = be32(dtb, 8) as usize; + let mut stack: Vec)>> = Vec::new(); + loop { + let token = be32(dtb, at); + at += 4; + match token { + 1 => { + let end = dtb[at..].iter().position(|&b| b == 0).unwrap(); + at = (at + end + 1).next_multiple_of(4); + stack.push(Vec::new()); + } + 2 => { + let props = stack.pop().unwrap(); + let hit = props.iter().any(|(n, v)| { + n == "compatible" && v.split(|&b| b == 0).any(|s| s == compatible) + }); + if hit { + return Some(props); + } + } + 3 => { + let len = be32(dtb, at) as usize; + let name_off = strings + be32(dtb, at + 4) as usize; + let name_end = dtb[name_off..].iter().position(|&b| b == 0).unwrap(); + let name = + String::from_utf8(dtb[name_off..name_off + name_end].to_vec()).unwrap(); + let value = dtb[at + 8..at + 8 + len].to_vec(); + at = (at + 8 + len).next_multiple_of(4); + stack.last_mut().unwrap().push((name, value)); + } + 4 => {} + 9 => return None, + _ => panic!("fdt token {token}"), + } + } + } + + /// Builder on the saved device tree == the private file's Globals and Power, byte for byte. + /// Needs `G15_J516C_FIXTURES=` with `j516c.dtb` (`/sys/firmware/fdt` of the machine) + /// and `t6031-j516c-board.bin` (its board file); both stay off the tree, so the test is + /// ignored by default (`--ignored` runs it) and fails, rather than passes, without them. + #[test] + #[ignore = "needs the off-tree J516C fixtures (G15_J516C_FIXTURES)"] + fn j516c_fixture() { + let dir = std::env::var_os("G15_J516C_FIXTURES") + .expect("G15_J516C_FIXTURES names the directory with j516c.dtb and the board file"); + let dir = std::path::Path::new(&dir); + let dtb = std::fs::read(dir.join("j516c.dtb")).unwrap(); + let blob = std::fs::read(dir.join("t6031-j516c-board.bin")).unwrap(); + + // The board file: magic, version, count, 3 x (kind, size), images (board/blob.rs). + assert_eq!(&blob[..8], b"T6031BRD"); + assert_eq!(get32(&blob, 8), 1); + assert_eq!(get32(&blob, 12), 3); + let mut at = 16 + 3 * 8; + let mut images = Vec::new(); + for i in 0..3 { + assert_eq!(get32(&blob, 16 + i * 8) as usize, i); + let size = get32(&blob, 16 + i * 8 + 4) as usize; + images.push(blob[at..at + size].to_vec()); + at += size; + } + assert_eq!(at, blob.len()); + let (hardware, file_globals, file_power) = (&images[0], &images[1], &images[2]); + assert_eq!(hardware.len(), 0xb480); + assert_eq!(file_globals.len(), GLOBALS_SIZE); + assert_eq!(file_power.len(), POWER_SIZE); + + // The GPU node's words. + let props = fdt_node_props(&dtb, b"apple,agx-t6031").expect("a T6031 GPU node"); + let prop = |name: &core::ffi::CStr| -> Option<&[u8]> { + let name = name.to_str().unwrap(); + props + .iter() + .find(|(n, _)| n == name) + .map(|(_, v)| v.as_slice()) + }; + let mut values = [0u32; PROPS.len()]; + for (v, name) in values.iter_mut().zip(PROPS.iter()) { + let bytes = prop(name).unwrap_or_else(|| panic!("GPU node has no {name:?}")); + assert_eq!(bytes.len(), 4, "{name:?}"); + *v = be32(bytes, 0); + } + let zones: Vec = prop(ZONES) + .map(|b| b.chunks(4).map(|c| be32(c, 0)).collect()) + .unwrap_or_default(); + let words = PowerWords::new(values, &zones).unwrap(); + + // The images: the G15S templates with the J516C words, the hardware record's maximum + // power and the top state. + let max_power = get32(hardware, HWDATA_A + HWDATA_A_MAX_POWER); + let mut built_globals = vec![0u8; GLOBALS_SIZE]; + super::super::template::fill(&super::super::data::GLOBALS, &mut built_globals).unwrap(); + globals(&mut built_globals, max_power, &words).unwrap(); + let mut built_power = vec![0u8; POWER_SIZE]; + super::super::template::fill(&super::super::data::POWER, &mut built_power).unwrap(); + power(&mut built_power, max_power, &words).unwrap(); + + assert_eq!( + first_difference(&built_globals, file_globals), + None, + "Globals" + ); + assert_eq!(first_difference(&built_power, file_power), None, "Power"); + } +} diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index 03b305c88e105..b3461d62bd575 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -2,14 +2,16 @@ //! Admission and firmware start to READY. //! -//! Admission (no firmware-visible writes): T6030 board check (`apple_pmp_t6030_board`), PMP -//! supplier (`apple_pmp_j514s_gpu_prepare`, may return EPROBE_DEFER), ADT regions and segments -//! ([`super::adt`]), exact firmware TEXT identity ([`super::identity`]), MMIO windows, runtime-PM -//! hold of the GFX domain. +//! Admission (no firmware-visible writes): machine check (the T6030 board check +//! `apple_pmp_t6030_board`, or the SoC's machine compatible), the PMP supplier on a SoC with +//! the PMP AGX vote (`apple_pmp_j514s_gpu_prepare`, may return EPROBE_DEFER), the GPU regions and +//! segments ([`super::adt`]: the ADT export, or the GPU device tree node), exact firmware TEXT +//! identity ([`super::identity`]), MMIO windows, runtime-PM hold of the GFX domain. //! -//! Start: PMP AGX vote (`apple_pmp_j514s_gpu_get`), hardware identity and core masks, `sgx_open` +//! Start: PMP AGX vote (`apple_pmp_j514s_gpu_get`) where the SoC takes one, hardware identity +//! and core masks (one word, or two above 32 core slots), `sgx_open` //! 0x70001, cold-only check (ASC RUN clear, TTBAT all zero), UAT adoption and the PTE self-check -//! (nothing firmware-visible yet), RTKit init (mailbox "mbox"), wake, endpoints 0x20/0x21, +//! (nothing firmware-visible yet), RTKit init (the node's mailbox), wake, endpoints 0x20/0x21, //! PPL/UAT bootstrap ([`super::uat`]), initdata graph ([`super::initdata`]), EP 0x20 //! `0x0081 << 48 | ROOT`, DEVCTL 0x13 then 9, readiness (`Runtime+0x314 == 0xabcdabcd` and //! `Flags+4 == 1`, 2 s, draining rings), DEVCTL 0x13 again. @@ -56,10 +58,12 @@ pub(crate) mod glue { extern "C" { fn asahi_g15_sha256(data: *const u8, len: usize, out: *mut u8); fn asahi_g15_board() -> bool; + fn asahi_g15_machine(compatible: *const core::ffi::c_char) -> bool; fn asahi_g15_hold_power(dev: *mut core::ffi::c_void); fn asahi_g15_module_get(); fn asahi_g15_adt_phram(start: *mut u64, size: *mut u64) -> core::ffi::c_int; fn asahi_g15_ram_disjoint(start: u64, size: u64) -> bool; + fn asahi_g15_ram_within(start: u64, size: u64) -> bool; fn asahi_g15_ioremap_wc(start: u64, size: usize) -> *mut core::ffi::c_void; fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } @@ -78,6 +82,12 @@ pub(crate) mod glue { unsafe { asahi_g15_board() } } + /// Whether the machine carries `compatible` (`of_machine_is_compatible()`). + pub(crate) fn machine(compatible: &CStr) -> bool { + // SAFETY: Reads the machine compatible strings only; the string is NUL-terminated. + unsafe { asahi_g15_machine(compatible.as_char_ptr()) } + } + /// Holds the GFX power domain for the retained session. pub(crate) fn hold_power(dev: &device::Device) { // SAFETY: `dev` is a valid device; runtime-PM accounting on it has no other requirement. @@ -113,6 +123,12 @@ pub(crate) mod glue { unsafe { asahi_g15_ram_disjoint(start, size) } } + /// The physical range lies entirely in System RAM. + pub(crate) fn ram_within(start: u64, size: u64) -> bool { + // SAFETY: Only queries the resource tree. + unsafe { asahi_g15_ram_within(start, size) } + } + /// Whether the device's DMA mappings skip cache maintenance. pub(crate) fn dma_coherent(dev: &device::Device) -> bool { // SAFETY: Reads a flag of a valid device. @@ -264,18 +280,20 @@ unsafe impl Send for Rtkit {} unsafe impl Sync for Rtkit {} impl Rtkit { - /// Initializes RTKit on mailbox "mbox" (not device-managed: retained). - fn new(dev: &device::Device, shared: &Arc) -> Result { + /// Initializes RTKit on the GPU node's mailbox (not device-managed: retained): the one + /// named "mbox" where the node names its mailboxes (the synthesized T6030 node), else the + /// first. + fn new(dev: &device::Device, shared: &Arc, named: bool) -> Result { let cookie = Arc::into_raw(shared.clone()) as *mut c_void; - // SAFETY: `dev` is valid, the name is a NUL-terminated string and the ops are static. + let name = if named { + c_str!("mbox").as_char_ptr() + } else { + core::ptr::null() + }; + // SAFETY: `dev` is valid, the name is a NUL-terminated string or null and the ops are + // static. let rtk = from_err_ptr(unsafe { - bindings::apple_rtkit_init( - dev.as_raw(), - cookie, - c_str!("mbox").as_char_ptr(), - 0, - &RTKIT_OPS, - ) + bindings::apple_rtkit_init(dev.as_raw(), cookie, name, 0, &RTKIT_OPS) }); match rtk { // INVARIANT: The cookie stays leaked from here on. @@ -382,7 +400,7 @@ pub(crate) struct Orphan { impl Orphan { /// Drains the receive rings, if they exist. pub(crate) fn drain(&mut self) -> Result { - if let Some(channels) = &self.channels { + if let Some(channels) = &mut self.channels { channels.drain_rings(&self.session.dev, &mut self.events)?; } Ok(()) @@ -397,24 +415,38 @@ pub(crate) fn admit( res: shared_regs::Resources, ) -> Result { let dev = pdev.as_ref(); - if !glue::board() { - return Err(ENODEV); - } - // Retain the PMP supplier (and its AGX bridge) before any vote. - // SAFETY: `dev` is the bound GPU device. - let ret = unsafe { bindings::apple_pmp_j514s_gpu_prepare(dev.as_raw()) }; - if ret != 0 { - let err = Error::from_errno(ret); - if err != EPROBE_DEFER { - dev_err!(dev, "PMP supplier not ready: {:?}\n", err); + let pmp = cfg.platform == hw::Platform::AdtWithPmp; + if pmp { + if !glue::board() { + return Err(ENODEV); + } + // Retain the PMP supplier (and its AGX bridge) before any vote. + // SAFETY: `dev` is the bound GPU device. + let ret = unsafe { bindings::apple_pmp_j514s_gpu_prepare(dev.as_raw()) }; + if ret != 0 { + let err = Error::from_errno(ret); + if err != EPROBE_DEFER { + dev_err!(dev, "PMP supplier not ready: {:?}\n", err); + } + return Err(err); } - return Err(err); + } else if !glue::machine(cfg.machine) { + dev_err!(dev, "G15: not a {} machine\n", cfg.name); + return Err(ENODEV); } - let adt = - adt::read(dev).inspect_err(|e| dev_err!(dev, "ADT GPU regions not admitted: {:?}\n", e))?; + let adt = adt::read(dev, cfg) + .inspect_err(|e| dev_err!(dev, "GPU regions not admitted: {:?}\n", e))?; identity::firmware(dev, &adt, cfg.firmware) .inspect_err(|e| dev_err!(dev, "loaded GFX TEXT identity mismatch: {:?}\n", e))?; dev_info!(dev, "M3: firmware identity accepted\n"); + if cfg.board == hw::Board::None { + dev_err!( + dev, + "G15: no board data for the {}; the firmware is left as the boot loader left it\n", + cfg.name + ); + return Err(ENODEV); + } // Firmware objects are cleaned to the point of coherency by their DMA mapping: that needs // a device that is not DMA-coherent, as the synthesized node is. if glue::dma_coherent(dev) { @@ -427,7 +459,7 @@ pub(crate) fn admit( let regs = regs::Regs::new(pdev, res)?; // Keep the GFX domain powered for the whole (retained) session. glue::hold_power(dev); - dev_info!(dev, "T6030 G15S admitted, ASC={:#x}\n", regs.asc_control()?); + dev_info!(dev, "{} admitted, ASC={:#x}\n", cfg.name, regs.asc_control()?); Ok(Admitted { regs, adt }) } @@ -445,6 +477,7 @@ fn devctl_op( /// From the wake to READY. On failure the caller keeps everything as an [`Orphan`]. fn run( session: &mut Session, + cfg: &Config, objects: &mut Objects, events: &mut Events, built: &mut Option<(InitData, Channels, u32)>, @@ -460,7 +493,7 @@ fn run( session.rtkit.start_endpoint(hw::ep::DOORBELL)?; let uat = session.uat.clone(); objects.window.bootstrap(&dev, &uat)?; - let (init, channels, _) = built.insert(initdata::build(&dev, &session.adt, objects)?); + let (init, channels, _) = built.insert(initdata::build(&dev, cfg, &session.adt, objects)?); uat::tlbi_all(); dev_info!( dev, @@ -520,16 +553,19 @@ pub(crate) fn start( ) -> Result { let Admitted { regs, adt } = admitted; let d: ARef = shared.dev.clone(); - // SAFETY: `d` is the bound GPU device. - let ret = unsafe { bindings::apple_pmp_j514s_gpu_get(d.as_raw()) }; - if ret != 0 { - dev_err!(d, "PMP AGX vote failed: {}\n", ret); - return Err(Error::from_errno(ret)); + if cfg.platform == hw::Platform::AdtWithPmp { + // SAFETY: `d` is the bound GPU device. + let ret = unsafe { bindings::apple_pmp_j514s_gpu_get(d.as_raw()) }; + if ret != 0 { + dev_err!(d, "PMP AGX vote failed: {}\n", ret); + return Err(Error::from_errno(ret)); + } } - let id = regs.identity()?; + let id = regs.identity(cfg)?; dev_info!( d, - "M3 G15S: power acknowledged, version={:#x} counts={:#x} core-mask={:#x}\n", + "{}: power acknowledged, version={:#x} counts={:#x} core-mask={:#x}\n", + cfg.name, id.version, id.counts, id.cores @@ -541,22 +577,32 @@ pub(crate) fn start( { dev_err!( d, - "unsupported G15S hardware: version {:#x} counts {:#x}\n", + "unsupported {} hardware: version {:#x} counts {:#x}\n", + cfg.name, id.version, id.counts ); return Err(eproto()); } - // Binning: any non-empty enabled set in each of the two clusters. - if id.core_masks[0] == 0 || id.core_masks[1] == 0 || id.cores >> (2 * hw::CLUSTER_BITS) != 0 { - dev_err!(d, "implausible G15S core mask {:#x}\n", id.cores); + // Binning: any non-empty enabled set in each cluster, nothing above the core slots. + let clusters = cfg.num_clusters as usize; + let slots = cfg.core_slots(); + if clusters > regs::MAX_CLUSTERS + || id.core_masks[..clusters].iter().any(|m| *m == 0) + || (slots < 64 && id.cores >> slots != 0) + { + dev_err!(d, "implausible {} core mask {:#x}\n", cfg.name, id.cores); return Err(eproto()); } + let mut enabled = [0u32; regs::MAX_CLUSTERS]; + for (count, mask) in enabled.iter_mut().zip(id.core_masks.iter()) { + *count = mask.count_ones(); + } dev_info!( d, - "M3 G15S: {} + {} cores enabled\n", - id.core_masks[0].count_ones(), - id.core_masks[1].count_ones() + "{}: cores enabled per cluster: {:?}\n", + cfg.name, + &enabled[..clusters] ); regs.open()?; // Cold only: ASC RUN clear and an all-zero TTBAT before ownership. @@ -580,7 +626,7 @@ pub(crate) fn start( // From here on the firmware is touched: nothing it can reach is freed. glue::retain_module(); - let rtkit = match Rtkit::new(&d, shared) { + let rtkit = match Rtkit::new(&d, shared, cfg.platform == hw::Platform::AdtWithPmp) { Ok(rtkit) => rtkit, Err(e) => { core::mem::forget(objects); @@ -601,7 +647,7 @@ pub(crate) fn start( let result = events .reserve() .and_then(|_| session.regs.set_asc_control(control | regs::ASC_RUN)) - .and_then(|_| run(&mut session, &mut objects, &mut events, &mut built)); + .and_then(|_| run(&mut session, cfg, &mut objects, &mut events, &mut built)); if let Err(e) = result { dev_err!( d, diff --git a/drivers/gpu/drm/asahi/g15/channel.rs b/drivers/gpu/drm/asahi/g15/channel.rs index 95dbfccba3a07..7935e1b38712d 100644 --- a/drivers/gpu/drm/asahi/g15/channel.rs +++ b/drivers/gpu/drm/asahi/g15/channel.rs @@ -12,7 +12,8 @@ //! without it, it does not wait (the kill request). Both return EBUSY while Control+0x4590 //! says the firmware is halted, and wait for an earlier posted record's consumption first. //! - Receive rings (event, 6 log sub-rings, trace, stats): consumed by setting read = producer; -//! the event ring is decoded by [`super::event`]. +//! the event ring is decoded by [`super::event`], the stats ring's tags and power-state +//! records are noted ([`super::power::pstate`]) before it is consumed. //! - FwCtl ring: 256 x 0x18 = 0x1800 bytes (the 2419 firmware uses a 0x18 stride). Never used //! by the host: no firmware cache flush path exists. @@ -30,7 +31,8 @@ use super::fw::channels::{ use super::hw::{self, rings, va}; use super::initdata::InitData; use super::object::{Class, FwObject, Objects, Placement, Request}; -use super::regs::wmb; +use super::power::pstate::{self, StatsSeen}; +use super::regs::{rmb, wmb}; use super::Firmware; /// A submit channel. @@ -74,6 +76,8 @@ pub(crate) struct Channels { pub(crate) log: Ring, pub(crate) trace: Ring, pub(crate) stats: Ring, + /// What the stats ring carried, up to its consumer. + pub(crate) stats_seen: StatsSeen, /// Allocated for the firmware; the host never uses it. _fwctl: Ring, /// Host producers of channels 0 TA, 1 FS, 2 compute. @@ -167,6 +171,7 @@ impl Channels { va::FWCTL_RING, rings::FWCTL_RING_SIZE, )?, + stats_seen: StatsSeen::default(), producer: [0; 3], devctl_producer: 0, }) @@ -241,14 +246,53 @@ impl Channels { Ok(()) } + /// Notes the stats records from the consumer to the producer into `seen`; returns the + /// producer, or `None` when an index is out of the ring. + fn note_stats(ring: &Ring, seen: &mut StatsSeen) -> Result> { + let producer = ring.state.r32(STATE_WRITE)?; + let read = ring.state.r32(STATE_READ)?; + let Some(slots) = pstate::pending(read, producer) else { + return Ok(None); + }; + rmb(); + for slot in slots { + let at = slot as usize * pstate::STATS_RECORD; + seen.note( + ring.ring.r32(at + pstate::STATS_TAG)?, + ring.ring.r32(at + pstate::POWER_STATE_INDEX)?, + ); + } + Ok(Some(producer)) + } + + /// The stats ring noted up to its producer, without consuming it. + pub(crate) fn peek_stats(&self) -> Result { + let mut seen = self.stats_seen; + Self::note_stats(&self.stats, &mut seen)?; + Ok(seen) + } + /// Drains every receive ring: events (decoded into `events`), the six log sub-rings, trace - /// and stats. - pub(crate) fn drain_rings(&self, dev: &kernel::device::Device, events: &mut Events) -> Result { + /// and stats (noted into [`Self::stats_seen`] first). + pub(crate) fn drain_rings( + &mut self, + dev: &kernel::device::Device, + events: &mut Events, + ) -> Result { events.drain(dev, &self.event)?; for i in 0..rings::LOG_RINGS { Self::drain_one(&self.log, i * 0x30)?; } Self::drain_one(&self.trace, 0)?; + let mut seen = self.stats_seen; + if let Some(producer) = Self::note_stats(&self.stats, &mut seen)? { + self.stats_seen = seen; + if self.stats.read()? != producer { + self.stats.state.w32(STATE_READ, producer)?; + wmb(); + } + return Ok(()); + } Self::drain_one(&self.stats, 0) } } diff --git a/drivers/gpu/drm/asahi/g15/encode.rs b/drivers/gpu/drm/asahi/g15/encode.rs index 95e418f34cc0d..e802509b4b866 100644 --- a/drivers/gpu/drm/asahi/g15/encode.rs +++ b/drivers/gpu/drm/asahi/g15/encode.rs @@ -83,6 +83,8 @@ pub(crate) const fn compact(gva: u64) -> u64 { /// Addresses the compute encoder names: compute graph and compute slot buffers. #[derive(Copy, Clone, Default)] pub(crate) struct ComputeRefs { + /// HardwareData (firmware VA): the activity records the sequence names. + pub(crate) hwdata: u64, /// Graph: notifier, queue info, user/firmware stamps, UMA manager (firmware VAs). pub(crate) notifier: u64, pub(crate) qinfo: u64, @@ -105,6 +107,7 @@ pub(crate) struct ComputeRefs { #[allow(clippy::too_many_arguments)] pub(crate) fn encode_compute( r: &ComputeRefs, + regs_cfg: &hw::ComputeRegs, c: &uapi::drm_asahi_cmd_compute, att: &[uapi::drm_asahi_attachment], ts_va: &[u64; 2], @@ -114,20 +117,28 @@ pub(crate) fn encode_compute( w: &mut [u8; compute::WORK_SIZE], q: &mut [u8; compute::SEQ_SIZE], ) { - let (wv, qv, h, pre) = (r.work, r.qinfo, hw::va::HWDATA, r.preempt_gva); + let (wv, qv, h, pre) = (r.work, r.qinfo, r.hwdata, r.preempt_gva); let t = stamp(n); let att_count = att.len().min(hw::sizes::MAX_ATTACHMENTS); + // The saved-state pointers into the preemption buffer and the two 0x1c words; a SoC + // encoded "fresh" carries zero in all six, as a first command does. + let (saved, word) = if regs_cfg.fresh { + (0, 0) + } else { + (pre + regs_cfg.tail, 0x1c) + }; + let step = |i: u64| if saved == 0 { 0 } else { saved + 8 * i }; let val: [u64; 22] = [ pre, c.cdm_ctrl_stream_base, - pre + 0x1480, - pre + 0x1488, - pre + 0x1490, - pre + 0x1498, + step(0), + step(1), + step(2), + step(3), 0x154024201, 0x10c08860, - 0x1c, - 0x1c, + word, + word, 0, 0, 0, @@ -136,10 +147,10 @@ pub(crate) fn encode_compute( 0, 0, 0, - 8, + regs_cfg.reg_1a0e9, 0xff0000, - 0x13200400020, - 0x200000001, + regs_cfg.ptc, + regs_cfg.dupm, ]; w.fill(0); @@ -489,6 +500,8 @@ pub(crate) fn barrier( /// (the UMA/PB objects are in `Pools`). #[derive(Copy, Clone, Default)] pub(crate) struct RenderRefs { + /// HardwareData (firmware VA): the activity records the sequences name. + pub(crate) hwdata: u64, pub(crate) notifier: u64, pub(crate) legacy_empty: u64, /// Queue info, user and firmware stamps of the TA (0) and FS (1) queues. @@ -542,7 +555,7 @@ pub(crate) fn encode_ta( w: &mut [u8; ta::WORK_SIZE], q: &mut [u8; ta::SEQ_SIZE], ) { - let (wv, uv, qv, h) = (p.ta_work, p.ta_seq, r.qinfo[0], hw::va::HWDATA); + let (wv, uv, qv, h) = (p.ta_work, p.ta_seq, r.qinfo[0], r.hwdata); let (heap, su) = (p.heap_gva, p.scene_user_gva); let ts = ts_va[0] != 0 || ts_va[1] != 0; let mut val: [u64; 72] = [ @@ -742,7 +755,7 @@ pub(crate) fn encode_fs( w: &mut [u8; fs::WORK_SIZE], q: &mut [u8; fs::SEQ_SIZE], ) { - let (wv, uv, qv, h) = (p.fs_work, p.fs_seq, r.qinfo[1], hw::va::HWDATA); + let (wv, uv, qv, h) = (p.fs_work, p.fs_seq, r.qinfo[1], r.hwdata); let (heap, su) = (p.heap_gva, p.scene_user_gva); let ts = ts_va[2] != 0 || ts_va[3] != 0; let (d, s) = (&u.depth, &u.stencil); diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 530c172556a7a..b61af1fb25a59 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -393,6 +393,8 @@ pub(crate) struct Engine { pub(crate) prof: Prof, /// The `asahi-g15/stats` file. pub(crate) debugfs: Option>>>>, + /// The `asahi-g15/pstate` file. + pub(crate) debugfs_pstate: Option>>>>, scratch: Option>, } @@ -427,6 +429,7 @@ impl Engine { stats: Stats::default(), prof: Prof::default(), debugfs: None, + debugfs_pstate: None, scratch: None, } } @@ -512,8 +515,12 @@ impl Firmware { if self.engine.flights.capacity() < sizes::FLIGHTS { self.engine.flights.reserve(sizes::FLIGHTS, GFP_KERNEL)?; } + // A job arriving at an idle engine: the PB reclaim, if due, happens before its + // publication (idle polls are not re-armed, so this is where a long idle ends). + self.reclaim_check(); self.engine.pending.push(job.clone(), GFP_KERNEL)?; job.progress.set_pending(true); + self.note_idle(); let q = &job.queue; let owned = self.engine.queue_published(&job) + self @@ -598,6 +605,9 @@ impl Firmware { } } } + // A publication that left nothing published and nothing pending (jobs that failed + // before any visible record) starts the idle. + self.note_idle(); result } @@ -638,6 +648,7 @@ impl Firmware { let r = &self.graph.render; let a = self.pools.allocator_refs(); RenderRefs { + hwdata: self.cfg.layout.hwdata_va, notifier: r.notifier.va, legacy_empty: r.legacy_empty.va, qinfo: [r.qinfo[0].va, r.qinfo[1].va], @@ -681,6 +692,8 @@ impl Firmware { } fn publish_prefix_into(&mut self, job: &Arc, f: &mut Flight) -> Result { + // Every backed PB chunk is mapped in the job's VM before anything of it is visible. + self.map_heap(&job.vm)?; let overlap = !self.engine.flights.is_empty(); let mut mixed = tuning::MIXED_PREFIX; let mut pf = Prefix { @@ -1085,6 +1098,7 @@ impl Firmware { let s = &scrub.compute[index]; let g = &self.graph.compute; let refs = ComputeRefs { + hwdata: self.cfg.layout.hwdata_va, notifier: g.notifier.va, qinfo: g.qinfo.va, user_stamp: g.user_stamp.va, @@ -1108,6 +1122,7 @@ impl Firmware { let (w, q) = (&mut scratch.cw, &mut scratch.cq); encode::encode_compute( &refs, + &self.cfg.compute, &cmd.cmd, &cmd.attachments, ts, diff --git a/drivers/gpu/drm/asahi/g15/fw/buffer.rs b/drivers/gpu/drm/asahi/g15/fw/buffer.rs index 4008aab2caa5a..777aa746c67be 100644 --- a/drivers/gpu/drm/asahi/g15/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/g15/fw/buffer.rs @@ -2,7 +2,7 @@ //! Parameter-buffer and UMA allocator records of the 2419 firmware. //! -//! The PB is one 256 MiB heap of [`MAX_PAGES`] pages of 0x8000 bytes in [`MAX_BLOCKS`] blocks of +//! The PB is one 512 MiB heap of [`MAX_PAGES`] pages of 0x8000 bytes in [`MAX_BLOCKS`] blocks of //! four pages. The firmware sees it through the page list (u32 records `G >> 15` of compact //! addresses `G = gva - 0x1000000000`, one per page), the block list (8-byte records, the low //! word `G >> 15` of the block's first page) and the PB manager P: P+0x28 = 4 * blocks pages, @@ -21,8 +21,8 @@ pub(crate) const PAGE_BYTES: u64 = 0x8000; pub(crate) const PAGES_PER_BLOCK: u32 = 4; pub(crate) const BLOCK_BYTES: u64 = 0x20000; /// Blocks and pages of the mapped 256 MiB heap. -pub(crate) const MAX_BLOCKS: u32 = 2048; -pub(crate) const MAX_PAGES: u32 = 8192; +pub(crate) const MAX_BLOCKS: u32 = 4096; +pub(crate) const MAX_PAGES: u32 = 16384; /// Records of the 0x8000-byte block list: the truthful K. pub(crate) const BLOCK_LIST_RECORDS: u32 = 4096; /// Sizes of the page list and the block list. @@ -275,7 +275,7 @@ mod tests { assert_eq!(block_record(HEAP, 1), 0xc18004); assert_eq!(block_record(HEAP, 2047), 0xc18000 + 4 * 2047); // The last page of the 256 MiB heap. - assert_eq!(page_record(HEAP, MAX_PAGES - 1), 0xc18000 + 8191); + assert_eq!(page_record(HEAP, MAX_PAGES - 1), 0xc18000 + 16383); assert_eq!(block_record_offset(0), 0); assert_eq!(block_record_offset(1023), 8 * 1023); assert_eq!(block_record_offset(4096), 0); @@ -287,7 +287,7 @@ mod tests { ); // The page list holds every page of the heap. assert!(MAX_PAGES as usize * 4 <= PAGE_LIST_SIZE); - assert_eq!(MAX_BLOCKS as u64 * BLOCK_BYTES, 0x10000000); + assert_eq!(MAX_BLOCKS as u64 * BLOCK_BYTES, 0x20000000); } #[test] diff --git a/drivers/gpu/drm/asahi/g15/fw/graph.rs b/drivers/gpu/drm/asahi/g15/fw/graph.rs index 42b45d8741d03..d88b0fc0f6c15 100644 --- a/drivers/gpu/drm/asahi/g15/fw/graph.rs +++ b/drivers/gpu/drm/asahi/g15/fw/graph.rs @@ -93,10 +93,15 @@ pub(crate) mod manager { } /// Compute pool table rows (0x40 bytes each, `gva | 1 << 62`) and freelist seeds (u64 VAs of -/// 0x1000-byte units of the first eight pools). +/// 0x1000-byte units of every pool). Seeding only the first eight pools (8192 pages, 32 MiB) let +/// ten compute workgroups of more than ~576 threads run at once and hung the eleventh (ticket +/// malik-na/linux-aurora#69): such groups take their memory from this pool. pub(crate) mod pools { pub(crate) const TABLE_STRIDE: usize = 0x40; pub(crate) const TABLE_VALID: u64 = 1 << 62; pub(crate) const SEED_BYTES: u64 = 0x1000; - pub(crate) const SEEDED_POOLS: usize = 8; + pub(crate) const SEEDED_POOLS: usize = crate::g15::hw::sizes::POOLS; + /// Seeds in the freelist. + pub(crate) const SEEDS: u32 = + (SEEDED_POOLS as u64 * crate::g15::hw::sizes::POOL_SIZE / SEED_BYTES) as u32; } diff --git a/drivers/gpu/drm/asahi/g15/fw/initdata.rs b/drivers/gpu/drm/asahi/g15/fw/initdata.rs index 4b832ff5eb68a..d5a1f0188b368 100644 --- a/drivers/gpu/drm/asahi/g15/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g15/fw/initdata.rs @@ -135,6 +135,9 @@ pub(crate) mod hwdata { pub(crate) const COMPUTE_ACTIVITY: usize = 0x3740; /// HardwareDataA: the maintained S8 request words. pub(crate) const HWDATA_A: usize = 0x4580; + /// HardwareDataA word holding the machine's maximum GPU power in mW (Globals +0x78 and + /// Power +0xd0 repeat it). + pub(crate) const MAX_POWER_A: usize = 0x6a4; pub(crate) const FAULT_INFO: usize = 0x8900; } diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index ee730d61affd4..91b1e94ebbcf1 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * G15 (T6030, M3 Pro) helpers that have no Rust bindings: static inlines, + * G15 (T6030/T6031, M3 Pro/Max) helpers that have no Rust bindings: static inlines, * macros and THIS_MODULE. Built only with CONFIG_DRM_ASAHI_G15. */ #include @@ -17,10 +17,12 @@ void asahi_g15_sha256(const u8 *data, size_t len, u8 *out); bool asahi_g15_board(void); +bool asahi_g15_machine(const char *compatible); void asahi_g15_hold_power(struct device *dev); void asahi_g15_module_get(void); int asahi_g15_adt_phram(u64 *start, u64 *size); bool asahi_g15_ram_disjoint(u64 start, u64 size); +bool asahi_g15_ram_within(u64 start, u64 size); void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size); bool asahi_g15_dma_coherent(struct device *dev); @@ -36,6 +38,12 @@ bool asahi_g15_board(void) return apple_pmp_t6030_board(); } +/* Whether the machine carries this compatible (of_machine_is_compatible()). */ +bool asahi_g15_machine(const char *compatible) +{ + return of_machine_is_compatible(compatible) > 0; +} + /* * Keep the GFX domain powered for the whole (retained) session. Never released, * like the firmware session itself. @@ -100,6 +108,13 @@ bool asahi_g15_dma_coherent(struct device *dev) return dev_is_dma_coherent(dev); } +/* The range lies entirely in System RAM (a reserved-memory region the kernel maps). */ +bool asahi_g15_ram_within(u64 start, u64 size) +{ + return region_intersects(start, size, IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE) == + REGION_INTERSECTS; +} + /* The inherited RTKit buffer inside firmware DATA. */ void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size) { diff --git a/drivers/gpu/drm/asahi/g15/graph.rs b/drivers/gpu/drm/asahi/g15/graph.rs index 5b52419565446..0e64baa7df60f 100644 --- a/drivers/gpu/drm/asahi/g15/graph.rs +++ b/drivers/gpu/drm/asahi/g15/graph.rs @@ -2,8 +2,8 @@ //! The persistent compute and render graphs. //! -//! The compute graph holds 22 x 4 MiB compute pools, the freelist (8192 seeds), the pool table, -//! the manager, scheduling context, notifier, threshold, job list, queue info/state, ring, +//! The compute graph holds 22 x 4 MiB compute pools, the freelist (every pool's pages), the pool +//! table, the manager, scheduling context, notifier, threshold, job list, queue info/state, ring, //! queue scratch and the user and firmware stamps. The render graph holds the render queues, //! scheduling context, notifier, threshold, job list, the empty legacy record and InitBM. The //! compute slots and the render pass slots are [`super::slots`]; the render UMA/PB allocators @@ -194,7 +194,7 @@ impl ComputeGraph { fn init(&self) -> Result { let mut entry = 0u64; - // Freelist: the first eight 4 MiB pools in 0x1000 units. + // Freelist: every 4 MiB pool in 0x1000 units. for pool in self.pools.iter().take(pools::SEEDED_POOLS) { let gva = pool.gva.ok_or(EINVAL)?; let mut off = 0; @@ -215,8 +215,8 @@ impl ComputeGraph { m.w32(manager::ONE, 1)?; m.w64(manager::FREELIST, self.freelist.gva.ok_or(EINVAL)?)?; m.w32(manager::UNK_1C, manager::UNK_1C_VALUE)?; - m.w32(manager::SEEDS_A, sizes::FREELIST_SEED)?; - m.w32(manager::SEEDS_B, sizes::FREELIST_SEED)?; + m.w32(manager::SEEDS_A, pools::SEEDS)?; + m.w32(manager::SEEDS_B, pools::SEEDS)?; m.w64(manager::POOL_TABLE, self.pool_table.gva.ok_or(EINVAL)?)?; m.w32(manager::UNK_44, manager::UNK_44_VALUE)?; m.w64(manager::COUNTER, self.counter.va)?; diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index 32b6a26043f73..a56d1fb6959ee 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -1,11 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! T6030 (G15S) hardware and firmware-ABI constants (layout values are shared by several -//! modules; change one only together with its users). The tuned policy values are in -//! [`tuning`]. +//! G15 hardware and firmware-ABI constants (layout values are shared by several modules; change +//! one only together with its users). The qualified T6030 values are the module-level +//! constants; what differs per SoC is a [`super::Config`] row ([`t6030`], [`t6031`]). The tuned +//! policy values are in [`tuning`]. use core::ops::Range; +pub(crate) mod t6031; + /// Identification register values of a qualified G15S. pub(crate) struct HwId { /// SGX identity word (`sgx + 0xd04000`). @@ -24,12 +27,99 @@ pub(crate) struct FirmwareBuild { pub(crate) text_sha256: [u8; 32], } +/// Where a SoC's GPU regions, firmware segments and performance states come from, and how its +/// GPU is powered. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum Platform { + /// T6030: the boot loader's ADT export ([`super::adt`]), a GPU node synthesized by the PMP + /// handoff, and the PMP AGX vote through the J514S supplier. + AdtWithPmp, + /// The GPU node of the kernel device tree, filled by the boot loader (regions, firmware + /// segments and operating points); the GPU is powered by its power domain alone. + DeviceTree, +} + +/// The source of the board images (HardwareData, Globals, Power) of a SoC. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum Board { + /// The hash-checked T6030 templates ([`super::board`]). + Templates, + /// The three images built from the device tree ([`super::board::record`], + /// [`super::board::device_tree`]) and cross-checked against a firmware file holding the + /// machine's own ([`super::board::blob`], pinned by its SHA-256), which also supplies the + /// record's words the device tree does not carry ([`super::board::record::FROM_FILE`]); a + /// difference refuses the start. + Blob { + name: &'static kernel::str::CStr, + sha256: [u8; 32], + }, + /// No board data: admission refuses the SoC before anything firmware-visible. + None, +} + +/// Per-SoC placement and size of the initdata objects that differ between G15 firmware +/// builds; everything else keeps the canonical [`va`] values. +pub(crate) struct Layout { + /// Root object version word. + pub(crate) root_magic: u64, + /// HardwareData VA, size, ASC-cached prefix (`None`: the whole object), and the offset of + /// the fourth HardwareDataB view (fault information). + pub(crate) hwdata_va: u64, + pub(crate) hwdata_size: usize, + pub(crate) hwdata_cached: Option, + pub(crate) hwdata_view: usize, + /// HardwareDataB IOMapping records. + pub(crate) iomaps: &'static [IoMap], + /// For a record built from the device tree ([`super::board::record`]): per IOMapping + /// record, in `iomaps` order, the mapped bytes, the element bytes and whether the firmware + /// may write the window; and the unmapped slots whose flag word is nevertheless 1. + pub(crate) record_slots: &'static [(u32, u32, bool)], + pub(crate) record_empty_writable: &'static [u8], +} + +/// Compute command register values that scale with the cluster count. +pub(crate) struct ComputeRegs { + /// Offset of the four saved-state pointers in the preemption buffer. + pub(crate) tail: u64, + /// UMA PTC (0xa599) and DUPM (0xd411). + pub(crate) ptc: u64, + pub(crate) dupm: u64, + /// The 0x1a0e9 word. + pub(crate) reg_1a0e9: u64, + /// Encode every command the way the first command of a fresh queue is encoded: zero + /// saved-state pointers and zero 0x12091/0x101d9. + pub(crate) fresh: bool, +} + /// The T6030 configuration. pub(crate) mod t6030 { use super::super::Config; - use super::{FirmwareBuild, HwId}; + use super::{ComputeRegs, FirmwareBuild, HwId, Layout}; + + /// The qualified graph: the canonical VAs and sizes of [`super::va`] and + /// [`super::super::fw::initdata`]. + pub(crate) const LAYOUT: Layout = Layout { + root_magic: super::super::fw::initdata::root::MAGIC, + hwdata_va: super::va::HWDATA, + hwdata_size: super::super::fw::initdata::hwdata::SIZE, + hwdata_cached: Some(super::super::fw::initdata::hwdata::CACHED), + hwdata_view: super::super::fw::initdata::hwdata::FAULT_INFO, + iomaps: &super::IOMAPS, + record_slots: &[], + record_empty_writable: &[], + }; + + /// The two-cluster compute register values. + pub(crate) const COMPUTE: ComputeRegs = ComputeRegs { + tail: 0x1480, + ptc: 0x13200400020, + dupm: 0x200000001, + reg_1a0e9: 8, + fresh: false, + }; pub(crate) const CONFIG: Config = Config { + name: "T6030 G15S", chip_id: 0x6030, gpu_variant: crate::hw::GpuVariant::S, gpu_revision: 0x11, @@ -52,6 +142,11 @@ pub(crate) mod t6030 { 0x80, 0x02, 0x93, 0xd5, ], }], + machine: kernel::c_str!("apple,t6030"), + platform: super::Platform::AdtWithPmp, + layout: &LAYOUT, + board: super::Board::Templates, + compute: COMPUTE, }; } @@ -180,6 +275,7 @@ pub(crate) mod rings { } /// One HardwareDataB IOMapping record: firmware device alias of an MMIO range. +#[derive(Copy, Clone)] pub(crate) struct IoMap { pub(crate) slot: u8, pub(crate) pa: u64, @@ -360,8 +456,8 @@ pub(crate) mod client { pub(crate) const CT_STRIDE: u64 = 0x1000000; pub(crate) const CT_SIZE: u64 = 0x400000; pub(crate) const PB_HEAP: u64 = 0x70c0000000; - pub(crate) const PB_HEAP_SIZE: u64 = 0x10000000; - pub(crate) const PB_BLOCK_LIST: u64 = 0x70e0000000; + pub(crate) const PB_HEAP_SIZE: u64 = 0x20000000; + pub(crate) const PB_BLOCK_LIST: u64 = 0x70f0000000; /// Compact-window objects (32-bit compact pointers: `gva - 0x1000000000`). pub(crate) const PRE0_PAGE: u64 = 0x1006080000; pub(crate) const PRE1_PAGE: u64 = 0x1006098000; @@ -386,8 +482,8 @@ pub(crate) mod sizes { pub(crate) const CLUSTER_PB_PAGES: u32 = 0x3c; pub(crate) const MAX_ATTACHMENTS: usize = 16; pub(crate) const MAX_CLIENT: usize = 320; - pub(crate) const PB_PAGES: u32 = 8192; - pub(crate) const PB_BLOCKS: u32 = 2048; + pub(crate) const PB_PAGES: u32 = 16384; + pub(crate) const PB_BLOCKS: u32 = 4096; pub(crate) const PB_LIST_RECORDS: u32 = 4096; pub(crate) const PB_GROW_STEP: u32 = 10; /// Flights (published prefixes) in flight at most; the second must be of the same VM. @@ -418,6 +514,9 @@ pub(crate) mod engine { /// Tuned policy. Every value was chosen by measurement on hardware; only the ones in /// `params.rs` are tunable. pub(crate) mod tuning { + /// Firmware idle standby time, us, of a machine whose device tree does not provide one (the + /// T6030): with the template's 700 us the GPU powers down between works ~100 us apart. + pub(crate) const IDLE_STANDBY_US: u32 = 2500; /// Command objects (work images, microsequences, barriers, queue info, scheduling contexts, /// job lists, FW-only queue scratch) map ASC-cached (AttrIndex 0). pub(crate) const FW_CACHED: bool = true; @@ -450,10 +549,12 @@ pub(crate) mod tuning { pub(crate) const SLOT_ROTATE: bool = true; /// Flights per VM. pub(crate) const MAX_INFLIGHT: u32 = 2; - /// PB growth through event 7 / DEVCTL 7, initial and maximum blocks. + /// PB growth through event 7 / DEVCTL 7; the initial posting is `g15_pb_blocks` (64, the + /// native driver's initial PB on this machine). pub(crate) const PB_GROW: bool = true; - pub(crate) const PB_BLOCKS: u32 = super::sizes::PB_BLOCKS / 2; - pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS; + /// The most PB blocks that may be posted: one below the heap range's block count, so the + /// block-list producer (modulo K = 4096) never returns to 0. + pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS - 1; /// At the host deadline, arm the firmware kill. pub(crate) const HANG_PREEMPT: bool = true; /// Compute start-sequence MMU fault mask (unmapped stores absorbed). diff --git a/drivers/gpu/drm/asahi/g15/hw/t6031.rs b/drivers/gpu/drm/asahi/g15/hw/t6031.rs new file mode 100644 index 0000000000000..ebd7171db9e4b --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/hw/t6031.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T6031 (M3 Max, G15C: J514C, J516C) configuration. +//! +//! One die, four clusters of ten core slots. The firmware is the same RTKit 2419 generation as +//! the T6030's and the initdata graph keeps the T6030 canonical VAs; what differs is the SoC +//! identity, the 40-bit enabled-core mask (two words), the source of the GPU regions (the +//! device tree node the boot loader fills, not an ADT export), the GPU power (its power domain; +//! there is no PMP vote), the initdata root version, a longer HardwareData record (0xb480 +//! bytes, its fourth view at +0xb400) placed aligned and ASC-cached, the IOMapping table (17 +//! slots at the T6031 register blocks) and the board images, built from the device tree and +//! checked against this machine's firmware file, which supplies the record's non-device-tree +//! words. Every +//! value was measured on a J516C: the SGX identification +//! registers, the loaded firmware segment, the device tree the boot loader hands over and the +//! initdata of that machine. + +use super::super::Config; +use super::{FirmwareBuild, HwId, IoMap, Layout}; + +/// SGX register window (the `sgx` resource), physical. +const SGX: u64 = 0x4_0800_0000; +/// The die 0 interrupt controller. +const AIC: u64 = 0x2_9240_0000; + +/// Base of the IOMapping firmware VAs and the gap left between consecutive mappings. +const IOMAP_BASE: u64 = 0xfffffc2068000000; +const IOMAP_GAP: u64 = 0x4000; +const PAGE: u64 = 0x4000; + +/// Mappings in slot order: slot, physical address, mapped bytes (rounded up to pages below), +/// element bytes, writable. Slots 0 (Fender, 0x144000), 1 (AIC timer page), 2 (AIC software +/// interrupts), 3 (RGX), 7, 9 (metrology sensors), 11 (memory cache, eight 0xd8000 +/// instances), 12 (AIC banked registers), 18-20 (telemetry dashboard), 21, 24, 25 (ANE), 26 +/// (PMS metrology), 28 and 29 (GPU clock generator at `sgx + 0xe1c000`). The record carries +/// each slot's physical address, sizes and access flag. +const SLOTS: [(u8, u64, u64, u64, bool); 17] = [ + (0, SGX + 0xd0_0000, 0x14_4000, 0x14_4000, true), + (1, 0x2_0e10_0000, 0x4000, 0x4000, false), + (2, AIC + 0x1_4000, 0x4000, 0x4000, true), + (3, SGX, 0x2_0000, 0x2_0000, true), + (7, 0x2_9044_4000, 0x4000, 0x4000, false), + (9, SGX + 0xe0_8000, 0x8000, 0x8000, true), + (11, 0x2_2000_0000, 8 * 0xd_8000, 0xd_8000, true), + (12, AIC + 0x4_c000, 0x4000, 0x4000, false), + (18, 0x2_903d_0000, 0x4000, 0x4000, true), + (19, 0x2_903c_0000, 0x4000, 0x4000, false), + (20, 0x2_903d_8000, 0x4000, 0x4000, true), + (21, SGX + 0xeb_4000, 0x4000, 0x4000, true), + (24, SGX + 0x100_0000, 0x8000, 0x8000, true), + (25, 0x3_c945_c000, 0x4000, 0x4000, true), + (26, 0x2_9040_0000, 0x8000, 0x8000, false), + (28, 0x2_a4dc_0000, 0x2_4000, 0x2_4000, false), + (29, SGX + 0xe1_c000, 0x4000, 0x4000, false), +]; + +/// The record's per-slot sizes and access flags, in [`SLOTS`] order. +const fn record_slots() -> [(u32, u32, bool); 17] { + let mut out = [(0, 0, false); 17]; + let mut i = 0; + while i < SLOTS.len() { + let (_, _, total, element, writable) = SLOTS[i]; + out[i] = (total as u32, element as u32, writable); + i += 1; + } + out +} +const RECORD_SLOTS: [(u32, u32, bool); 17] = record_slots(); + +/// Unmapped slots whose IOMapping flag word the J516C record holds as 1. +const RECORD_EMPTY_WRITABLE: [u8; 9] = [4, 6, 8, 10, 13, 14, 15, 16, 17]; + +/// Packs the slots' firmware VAs: consecutive from [`IOMAP_BASE`], each mapping followed by one +/// unmapped page. +const fn iomaps() -> [IoMap; 17] { + let mut out = [IoMap { + slot: 0, + pa: 0, + map_bytes: 0, + va: 0, + delta: 0, + }; 17]; + let mut va = IOMAP_BASE; + let mut i = 0; + while i < SLOTS.len() { + let (slot, pa, bytes, _, _) = SLOTS[i]; + let delta = pa & (PAGE - 1); + let map_bytes = (delta + bytes + PAGE - 1) & !(PAGE - 1); + out[i] = IoMap { + slot, + pa: pa & !(PAGE - 1), + map_bytes, + va, + delta, + }; + va += map_bytes + IOMAP_GAP; + i += 1; + } + out +} + +/// HardwareDataB IOMapping records of the J516C. +pub(crate) const IOMAPS: [IoMap; 17] = iomaps(); + +/// The J516C initdata layout: the T6030 graph with the root version this firmware's host +/// driver sends and the longer HardwareData record, placed 16-byte aligned (the firmware's MTR +/// alarm handler does a 64-bit atomic in HardwareDataA; at the T6030 placement the record is +/// 4-byte aligned, which faulted on T8122) and mapped ASC-cached as a whole. +pub(crate) const LAYOUT: Layout = Layout { + root_magic: 0x0c08e21e84820490, + hwdata_va: 0xfffffc20406fb580, + hwdata_size: 0xb480, + hwdata_cached: None, + hwdata_view: 0xb400, + iomaps: &IOMAPS, + record_slots: &RECORD_SLOTS, + record_empty_writable: &RECORD_EMPTY_WRITABLE, +}; + +/// The J516C board file (`asahi/t6031-j516c-board.bin`, built into the kernel): the hardware +/// record, Globals and the power block that this machine's firmware accepted, as the earlier +/// J516C driver uploaded them (its device-tree-built records with the per-board words of this +/// machine applied). Kept out of the tree; pinned by its SHA-256. The three images are built +/// from the device tree and must match the file's; the file supplies the record's words the +/// device tree does not carry (`board::record::FROM_FILE`). +const BOARD_SHA256: [u8; 32] = [ + 0x4e, 0x03, 0xf2, 0x9f, 0xa4, 0x47, 0xdc, 0xeb, 0xfd, 0x4d, 0x1c, 0x22, 0x5a, 0x64, 0x7b, + 0xfa, 0x31, 0xb3, 0x63, 0x45, 0x8f, 0x28, 0x33, 0x82, 0x3a, 0x5c, 0xa8, 0xa1, 0x0e, 0xdf, + 0xf3, 0xfc, +]; + +pub(crate) const CONFIG: Config = Config { + name: "T6031 G15C", + chip_id: 0x6031, + gpu_variant: crate::hw::GpuVariant::C, + gpu_revision: 0x12, + uat_ias: 42, + uat_oas: 42, + num_clusters: 4, + num_cores_per_cluster: 10, + hw_ids: &[HwId { + version: 0x07041100, + counts: 0x0021040a, + }], + // The stubs the boot chains of this J516C leave loaded (RTKit-2419.140.12): the macOS 14.7 + // chain's and the macOS 14.8 boot volume's. Same segment sizes and marker; the layout, board + // images and initdata below do not depend on which one is loaded. A further build can be + // admitted for one boot with `g15_accept_text_prefix`, which logs its digest. + firmware: &[ + FirmwareBuild { + name: "macOS 14.7 stub firmware (iBoot-10151.140.19.700.2), J516C", + text_size: 0x64000, + data_size: 0x140000, + marker: 0x5cf28, + text_sha256: [ + 0xc2, 0x6e, 0xc8, 0x9c, 0x11, 0x50, 0xeb, 0x65, 0x49, 0xcf, 0x60, 0x2e, 0x01, + 0x2d, 0x0c, 0x5f, 0x49, 0xeb, 0x75, 0x6e, 0x89, 0x6d, 0x6b, 0xb8, 0x34, 0x2c, + 0xb2, 0xc5, 0xd9, 0x8b, 0x33, 0x2c, + ], + }, + FirmwareBuild { + name: "macOS 14.8 stub firmware, J516C", + text_size: 0x64000, + data_size: 0x140000, + marker: 0x5cf28, + text_sha256: [ + 0x1b, 0xf2, 0xfd, 0x34, 0x76, 0xbe, 0xfe, 0x73, 0x6b, 0xc2, 0xc0, 0x70, 0xba, + 0xd4, 0x5f, 0x8b, 0xe3, 0x6b, 0x68, 0xcd, 0x11, 0xbc, 0xff, 0x16, 0xc9, 0x02, + 0x9d, 0x6b, 0xcd, 0xa0, 0x26, 0xcf, + ], + }, + ], + machine: kernel::c_str!("apple,t6031"), + platform: super::Platform::DeviceTree, + layout: &LAYOUT, + board: super::Board::Blob { + name: kernel::c_str!("asahi/t6031-j516c-board.bin"), + sha256: BOARD_SHA256, + }, + // Four clusters: the saved-state pointers at +0x2980 of the preemption buffer (0xd80 plus + // 0x700 per cluster), UMA PTC 0x6000400020, DUPM 0x3c000000140 (32 N | 96 N << 32 for + // N = 10 cores per cluster) and 0x1a0e9 = 0, with the saved-state pointers and the + // 0x12091/0x101d9 words zero in every command, as this machine's driver encodes a queue's + // first command. Read from 38 compute commands on a J516C; the two-cluster values fault + // the first job on this GPU. + compute: super::ComputeRegs { + tail: 0x2980, + ptc: 0x6000400020, + dupm: 0x3c000000140, + reg_1a0e9: 0, + fresh: true, + }, +}; diff --git a/drivers/gpu/drm/asahi/g15/identity.rs b/drivers/gpu/drm/asahi/g15/identity.rs index 2a662c2c5f596..405fc97b844de 100644 --- a/drivers/gpu/drm/asahi/g15/identity.rs +++ b/drivers/gpu/drm/asahi/g15/identity.rs @@ -3,17 +3,27 @@ //! Exact firmware and hardware identity. //! //! The firmware build is the SHA-256 of TEXT with the 8 entropy bytes after the single -//! `GKTS\x08\0\0\0` marker zeroed, plus the segment sizes, looked up in -//! [`super::hw::t6030::CONFIG`]`.firmware`; the hardware revision is checked against its -//! `hw_ids` at start ([`super::boot`]). An unknown build or revision is refused with its measured -//! identity. The canonical object VAs, the initdata graph, the board templates and the work -//! layouts all belong to one build, so nothing is written before the build is identified. +//! `GKTS\x08\0\0\0` marker zeroed, plus the segment sizes, looked up in the SoC +//! configuration's `firmware` list; the hardware revision is checked against its `hw_ids` at +//! start ([`super::boot`]). An unknown build or revision is refused with its measured identity; +//! an unknown build's refusal also says what to report so that it can be pinned. The pure +//! lookup is in [`lookup`] (host-tested). +//! The canonical object VAs, the initdata graph, the board images and the work layouts all +//! belong to one build, so nothing is written before the build is identified. +//! +//! Bring-up escape hatch: `g15_accept_text_prefix=` admits an unpinned build whose digest +//! starts with those 8 bytes (the full digest and marker offset are logged, so that the build +//! can be pinned by the next kernel). use kernel::{bindings, device, fmt, prelude::*}; use super::boot::glue; use super::hw::FirmwareBuild; +mod lookup; + +use lookup::{Measured, Verdict}; + /// Lowercase hex of a byte string, as `%*phN` prints it. pub(crate) struct Hex<'a>(pub(crate) &'a [u8]); @@ -46,55 +56,70 @@ pub(crate) fn copy_physical(phys: u64, size: usize) -> Result> { Ok(copy?) } -/// The GKTS marker: the 8 bytes after it are per-boot entropy. -const MARKER: [u8; 8] = [0x47, 0x4b, 0x54, 0x53, 0x08, 0, 0, 0]; +impl From<&FirmwareBuild> for Measured { + fn from(build: &FirmwareBuild) -> Self { + Measured { + text_size: build.text_size, + data_size: build.data_size, + marker: build.marker, + sha256: build.text_sha256, + } + } +} -/// Identifies the loaded firmware build. +/// Identifies the loaded firmware build: the pinned build, or `None` for a build admitted by +/// its digest prefix (`g15_accept_text_prefix`). pub(crate) fn firmware( dev: &device::Device, adt: &super::adt::GpuAdt, builds: &'static [FirmwareBuild], -) -> Result<&'static FirmwareBuild> { +) -> Result> { let size = usize::try_from(adt.text.end - adt.text.start)?; let data_size = adt.data.end - adt.data.start; let mut copy = copy_physical(adt.text.start, size)?; - let mut at = 0usize; - let mut found = 0usize; - let mut off = 0usize; - while off + 16 <= size { - if copy[off..off + 8] == MARKER { - at = off; - found += 1; - } - off += 8; - } - if found != 1 { + let at = lookup::marker(©).map_err(|found| { dev_err!( dev, "GPU firmware TEXT has {} GKTS markers, need one\n", found ); - return Err(EINVAL); - } + EINVAL + })?; copy[at + 8..at + 16].fill(0); - let digest = glue::sha256(©); - for build in builds { - if build.text_size == size as u64 - && build.data_size == data_size - && build.marker as usize == at - && build.text_sha256 == digest - { + let m = Measured { + text_size: size as u64, + data_size, + marker: u32::try_from(at)?, + sha256: glue::sha256(©), + }; + let qualified = builds.iter().map(Measured::from); + match lookup::verdict(&m, qualified, super::params::accept_text_prefix()) { + Verdict::Pinned(i) => { + let build = builds.get(i).ok_or(EINVAL)?; dev_info!(dev, "GPU firmware build: {}\n", build.name); - return Ok(build); + return Ok(Some(build)); + } + Verdict::Admitted => { + dev_warn!( + dev, + "GPU firmware build admitted by g15_accept_text_prefix: TEXT {:#x} DATA {:#x} GKTS {:#x} sha256 {}\n", + size, + data_size, + at, + Hex(&m.sha256) + ); + return Ok(None); } + Verdict::Refused => {} } dev_err!( dev, - "unsupported GPU firmware build: TEXT {:#x} DATA {:#x} GKTS {:#x} sha256 {}\n", + "unsupported GPU firmware build: TEXT {:#x} DATA {:#x} GKTS {:#x} sha256 {}; {}\n", size, data_size, at, - Hex(&digest) + Hex(&m.sha256), + lookup::REPORT_HINT ); for build in builds { dev_err!(dev, "supported: {}\n", build.name); diff --git a/drivers/gpu/drm/asahi/g15/identity/lookup.rs b/drivers/gpu/drm/asahi/g15/identity/lookup.rs new file mode 100644 index 0000000000000..6de0eb314ddee --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/identity/lookup.rs @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pure firmware-build identification: the GKTS marker search, the lookup of a measured build in +//! the qualified list, the `g15_accept_text_prefix` escape and the refusal hint. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. + +/// The GKTS marker: the 8 bytes after it are per-boot entropy. +pub(crate) const MARKER: [u8; 8] = [0x47, 0x4b, 0x54, 0x53, 0x08, 0, 0, 0]; + +/// Appended to the refusal of an unknown build: what a user reports so that it can be pinned. +pub(crate) const REPORT_HINT: &str = + "to have it supported, report this line and the macOS version of the boot chain that loaded it"; + +/// The identity of a loaded firmware build: segment sizes, marker offset and the TEXT digest with +/// the marker's entropy zeroed. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Measured { + pub(crate) text_size: u64, + pub(crate) data_size: u64, + pub(crate) marker: u32, + pub(crate) sha256: [u8; 32], +} + +/// The outcome of identifying a measured build. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Verdict { + /// The qualified build at this index. + Pinned(usize), + /// Not qualified, admitted for this boot because its digest prefix was named. + Admitted, + /// Not qualified and not admitted. + Refused, +} + +/// Offsets of the 8-byte-aligned GKTS markers in `text` that have their 8 entropy bytes in +/// range: `Ok(offset)` for exactly one, `Err(count)` otherwise. +pub(crate) fn marker(text: &[u8]) -> Result { + let mut at = 0usize; + let mut found = 0usize; + let mut off = 0usize; + while off + 16 <= text.len() { + if text[off..off + 8] == MARKER { + at = off; + found += 1; + } + off += 8; + } + if found == 1 { + Ok(at) + } else { + Err(found) + } +} + +/// The first 8 digest bytes, big-endian, as `g15_accept_text_prefix` names them. +pub(crate) fn prefix(sha256: &[u8; 32]) -> u64 { + let mut first = [0u8; 8]; + first.copy_from_slice(&sha256[..8]); + u64::from_be_bytes(first) +} + +/// Looks `m` up in `qualified`; an unqualified build is admitted only when `accept_prefix` is +/// nonzero and equals its digest prefix. +pub(crate) fn verdict( + m: &Measured, + qualified: impl IntoIterator, + accept_prefix: u64, +) -> Verdict { + if let Some(i) = qualified.into_iter().position(|q| q == *m) { + return Verdict::Pinned(i); + } + if accept_prefix != 0 && accept_prefix == prefix(&m.sha256) { + Verdict::Admitted + } else { + Verdict::Refused + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn build(n: u8) -> Measured { + let mut sha256 = [0u8; 32]; + sha256[0] = 0x1b; + sha256[7] = n; + sha256[31] = n; + Measured { + text_size: 0x64000, + data_size: 0x140000, + marker: 0x5cf28, + sha256, + } + } + + #[test] + fn finds_one_marker() { + let mut text = [0u8; 64]; + text[16..24].copy_from_slice(&MARKER); + assert_eq!(marker(&text), Ok(16)); + } + + #[test] + fn refuses_none_two_or_misaligned_markers() { + let mut text = [0u8; 64]; + assert_eq!(marker(&text), Err(0)); + text[12..20].copy_from_slice(&MARKER); + assert_eq!(marker(&text), Err(0)); + text = [0u8; 64]; + text[8..16].copy_from_slice(&MARKER); + text[32..40].copy_from_slice(&MARKER); + assert_eq!(marker(&text), Err(2)); + } + + #[test] + fn needs_the_entropy_bytes_in_range() { + let mut text = [0u8; 64]; + text[56..64].copy_from_slice(&MARKER); + assert_eq!(marker(&text), Err(0)); + } + + #[test] + fn prefix_is_the_first_eight_bytes_big_endian() { + let mut sha256 = [0u8; 32]; + sha256[..8].copy_from_slice(&[0x1b, 0xf2, 0xfd, 0x34, 0x76, 0xbe, 0xfe, 0x73]); + sha256[8] = 0xff; + assert_eq!(prefix(&sha256), 0x1bf2fd3476befe73); + } + + #[test] + fn pins_the_matching_build() { + assert_eq!( + verdict(&build(2), [build(1), build(2)], 0), + Verdict::Pinned(1) + ); + } + + #[test] + fn pinned_wins_over_the_prefix() { + let m = build(1); + assert_eq!(verdict(&m, [m], prefix(&m.sha256)), Verdict::Pinned(0)); + } + + #[test] + fn every_field_must_match() { + let q = build(1); + let mut m = q; + m.text_size += 0x1000; + assert_eq!(verdict(&m, [q], 0), Verdict::Refused); + m = q; + m.data_size += 0x1000; + assert_eq!(verdict(&m, [q], 0), Verdict::Refused); + m = q; + m.marker += 8; + assert_eq!(verdict(&m, [q], 0), Verdict::Refused); + m = q; + m.sha256[31] ^= 1; + assert_eq!(verdict(&m, [q], 0), Verdict::Refused); + } + + #[test] + fn admits_only_the_named_prefix() { + let m = build(3); + assert_eq!( + verdict(&m, [build(1)], prefix(&m.sha256)), + Verdict::Admitted + ); + assert_eq!( + verdict(&m, [build(1)], prefix(&build(4).sha256)), + Verdict::Refused + ); + } + + #[test] + fn zero_prefix_admits_nothing() { + let m = Measured { + sha256: [0u8; 32], + ..build(0) + }; + assert_eq!(verdict(&m, [build(1)], 0), Verdict::Refused); + } + + #[test] + fn hint_says_what_to_report() { + assert!(REPORT_HINT.contains("report this line")); + assert!(REPORT_HINT.contains("macOS version")); + } +} diff --git a/drivers/gpu/drm/asahi/g15/initdata.rs b/drivers/gpu/drm/asahi/g15/initdata.rs index 59f2e5dc50cc7..6031806a9e9aa 100644 --- a/drivers/gpu/drm/asahi/g15/initdata.rs +++ b/drivers/gpu/drm/asahi/g15/initdata.rs @@ -27,6 +27,7 @@ use super::fw::initdata::{ use super::hw::{self, va}; use super::identity::Hex; use super::object::{Class, FwObject, Objects, Placement, Request}; +use super::Config; /// The built initdata graph. pub(crate) struct InitData { @@ -113,56 +114,13 @@ fn put64(b: &mut [u8], offset: usize, value: u64) -> Result { Ok(()) } -/// Builds the initdata graph. Returns it with its channels and the top DVFS frequency in kHz. -/// Nothing is published here. -pub(crate) fn build( + +/// The T6030 board images: the verified templates with DVFS from the ADT. Returns the images +/// and the top DVFS frequency in kHz. +fn templates( dev: &device::Device, adt: &GpuAdt, - objects: &mut Objects, -) -> Result<(InitData, Channels, u32)> { - let mut init = InitData::new(); - let fw = Class::Fw; - - // Owned objects and channels. - let root_obj = canonical(objects, va::ROOT, root::SIZE, fw)?; - let region_a_obj = canonical(objects, va::REGION_A, region_a::SIZE, fw)?; - let runtime_obj = canonical(objects, va::RUNTIME, runtime::SIZE, fw)?; - // Activity records cached; fault info and HardwareDataA timestamps not. - let hwdata_obj = objects.alloc_split( - Request { - size: hwdata::SIZE, - class: fw, - placement: Placement::At(va::HWDATA), - offset: 0, - client: None, - lower: None, - cpu: true, - }, - Some(( - (va::HWDATA as usize & (hw::PAGE as usize - 1)) + hwdata::CACHED, - if hw::tuning::HWDATA_CACHED { - Class::FwCached - } else { - fw - }, - )), - )?; - let pair_obj = canonical(objects, va::PAIR, pair::SIZE, fw)?; - let small_obj = canonical(objects, va::SMALL, small::SIZE, fw)?; - let c0 = canonical(objects, va::C0, cbuf::C0_SIZE, Class::Shared)?; - let c1 = canonical(objects, va::C1, cbuf::C1_SIZE, Class::Shared)?; - let c3 = canonical(objects, va::C3, cbuf::C3_SIZE, Class::Shared)?; - let globals_obj = canonical(objects, va::GLOBALS, globals::SIZE, fw)?; - let power_obj = canonical(objects, va::POWER, power::SIZE, fw)?; - let control_obj = canonical(objects, va::CONTROL, control::SIZE, fw)?; - let flags_obj = canonical(objects, va::FLAGS, flags::SIZE, fw)?; - let channels = Channels::alloc(objects)?; - // Firmware Device RW aliases for the IOMapping table. - for map in hw::IOMAPS.iter() { - objects.map_mmio(map.va, map.pa, map.map_bytes)?; - } - - // Board images, verified, then the owned replacements. +) -> Result<(KVVec, KVVec, KVVec, u32)> { let mut hw_img = KVVec::from_elem(0u8, hwdata::SIZE, GFP_KERNEL)?; let mut glob = KVVec::from_elem(0u8, globals::SIZE, GFP_KERNEL)?; let mut pwr = KVVec::from_elem(0u8, power::SIZE, GFP_KERNEL)?; @@ -196,8 +154,141 @@ pub(crate) fn build( if !reference { dev_info!(dev, "board data sha256 {}\n", Hex(&glue::sha256(&hw_img))); } + Ok((hw_img, glob, pwr, max_khz)) +} + +/// HardwareDataA word holding the system counter at initdata creation. +const HWDATA_A_INIT_TIMESTAMP: usize = 0x424c; + +/// Board images of a SoC with a firmware file. The hardware record ([`board::hardware_record`]), +/// Globals and Power ([`board::device_tree`]) are built from the device tree; the file, this +/// machine's own, supplies the record's words the device tree does not carry and is otherwise +/// a cross-check: the start is refused at the first differing word. Returns the images and the +/// top operating-point frequency in kHz. +fn blob( + dev: &device::Device, + cfg: &Config, + adt: &GpuAdt, + name: &CStr, + sha256: &[u8; 32], +) -> Result<(KVVec, KVVec, KVVec, u32)> { + let layout = cfg.layout; + let images = board::blob::load( + dev, + name, + sha256, + [layout.hwdata_size, globals::SIZE, power::SIZE], + )?; + let mut hw_img = board::hardware_record(dev, cfg, adt, &images.hardware)?; + let (glob, pwr, max_power_mw) = board::device_tree(dev, &hw_img)?; + for (what, built, file) in [ + ("hardware record", &hw_img, &images.hardware), + ("Globals", &glob, &images.globals), + ("Power", &pwr, &images.power), + ] { + let diff = if what == "hardware record" { + board::record::cross_check(built, file) + } else { + board::words::first_difference(built, file) + }; + if let Some((at, b, f)) = diff { + dev_err!( + dev, + "board data: device-tree {} differs from {} at +{:#x}: built {:#x}, file {:#x}\n", + what, + name, + at, + b, + f + ); + return Err(EINVAL); + } + } + let max_mhz = adt.perf_mhz.iter().copied().max().unwrap_or(0); + let now: u64; + // SAFETY: Reading the architectural counter has no side effects. + unsafe { + core::arch::asm!("mrs {}, cntpct_el0", out(reg) now, options(nomem, nostack, preserves_flags)) + }; + put64(&mut hw_img, hwdata::HWDATA_A + HWDATA_A_INIT_TIMESTAMP, now)?; + dev_info!( + dev, + "board data: {} images from the device tree match {} ({} record words from the file, {} file words left zero), {} mW maximum power, {} MHz top operating point\n", + cfg.name, + name, + board::record::FROM_FILE.len(), + board::record::UNNEEDED.len(), + max_power_mw, + max_mhz + ); + Ok((hw_img, glob, pwr, max_mhz * 1000)) +} + +/// Builds the initdata graph. Returns it with its channels and the top DVFS frequency in kHz. +/// Nothing is published here. +pub(crate) fn build( + dev: &device::Device, + cfg: &Config, + adt: &GpuAdt, + objects: &mut Objects, +) -> Result<(InitData, Channels, u32)> { + let mut init = InitData::new(); + let fw = Class::Fw; + let layout = cfg.layout; + let hwdata_va = layout.hwdata_va; + + // Owned objects and channels. + let root_obj = canonical(objects, va::ROOT, root::SIZE, fw)?; + let region_a_obj = canonical(objects, va::REGION_A, region_a::SIZE, fw)?; + let runtime_obj = canonical(objects, va::RUNTIME, runtime::SIZE, fw)?; + // Activity records cached; fault info and HardwareDataA timestamps not (T6030), or the + // whole record cached (a layout without a cached prefix). + let cached = if hw::tuning::HWDATA_CACHED { + Class::FwCached + } else { + fw + }; + let hwdata_obj = match layout.hwdata_cached { + Some(prefix) => objects.alloc_split( + Request { + size: layout.hwdata_size, + class: fw, + placement: Placement::At(hwdata_va), + offset: 0, + client: None, + lower: None, + cpu: true, + }, + Some(((hwdata_va as usize & (hw::PAGE as usize - 1)) + prefix, cached)), + )?, + None => canonical(objects, hwdata_va, layout.hwdata_size, cached)?, + }; + let pair_obj = canonical(objects, va::PAIR, pair::SIZE, fw)?; + let small_obj = canonical(objects, va::SMALL, small::SIZE, fw)?; + let c0 = canonical(objects, va::C0, cbuf::C0_SIZE, Class::Shared)?; + let c1 = canonical(objects, va::C1, cbuf::C1_SIZE, Class::Shared)?; + let c3 = canonical(objects, va::C3, cbuf::C3_SIZE, Class::Shared)?; + let globals_obj = canonical(objects, va::GLOBALS, globals::SIZE, fw)?; + let power_obj = canonical(objects, va::POWER, power::SIZE, fw)?; + let control_obj = canonical(objects, va::CONTROL, control::SIZE, fw)?; + let flags_obj = canonical(objects, va::FLAGS, flags::SIZE, fw)?; + let channels = Channels::alloc(objects)?; + // Firmware Device RW aliases for the IOMapping table. + for map in layout.iomaps.iter() { + objects.map_mmio(map.va, map.pa, map.map_bytes)?; + } + + // Board images, verified, then the owned replacements. + let (mut hw_img, mut glob, pwr, max_khz) = match cfg.board { + hw::Board::Templates => templates(dev, adt)?, + hw::Board::Blob { name, sha256 } => blob(dev, cfg, adt, name, &sha256)?, + hw::Board::None => return Err(ENODEV), + }; + if hw_img.len() != layout.hwdata_size { + return Err(EINVAL); + } put64(&mut hw_img, hwdata::TTBAT, adt.ttbs.start)?; - for map in hw::IOMAPS.iter() { + for map in layout.iomaps.iter() { put64( &mut hw_img, hwdata::IOMAPS + hwdata::IOMAP_STRIDE * map.slot as usize + hwdata::IOMAP_VA, @@ -206,11 +297,47 @@ pub(crate) fn build( } put64(&mut hw_img, hwdata::TIMESTAMPS, va::TIMESTAMPS)?; put32(&mut glob, globals::UNK_980, 100)?; - put32( - &mut glob, - globals::IDLE_STANDBY_US, - super::params::idle_standby_us(), - )?; + // The device tree's standby time (a machine with a firmware file; the templates' machine + // has none). + let dt_standby = match cfg.board { + hw::Board::Blob { .. } => { + let prop = board::words::PROPS[board::words::STANDBY_TIMER]; + match dev + .of_node() + .ok_or(ENODEV) + .and_then(|n| n.get_property::(prop)) + { + Ok(v) => Some(v), + Err(_) => { + dev_warn!( + dev, + "GPU node has no u32 {}, using the tuned idle standby\n", + prop + ); + None + } + } + } + _ => None, + }; + let (standby_us, standby_src, dt_rejected) = super::power::standby( + super::params::idle_standby_override_us(), + dt_standby, + hw::tuning::IDLE_STANDBY_US, + ); + if dt_rejected { + dev_warn!( + dev, + "apple,idleoff-standby-timer is outside 100..100000 us, ignored\n" + ); + } + dev_info!( + dev, + "idle standby {} us from {}\n", + standby_us, + standby_src.name() + ); + put32(&mut glob, globals::IDLE_STANDBY_US, standby_us)?; put32( &mut glob, globals::IDLE_MIN_CONF, @@ -221,16 +348,19 @@ pub(crate) fn build( globals::IDLE_PREDICTIVE, hw::tuning::IDLE_PREDICTIVE as u32, )?; - // HardwareDataA at +0x4580: maintained S8 request. - for (offset, value) in [ - (0x2c, 8), - (0x30, 8), - (0x44, 800), - (0x7d8, 800), - (0x7e0, 800), - (0x81c, 800), - ] { - put32(&mut hw_img, hwdata::HWDATA_A + offset, value)?; + // HardwareDataA at +0x4580: maintained S8 request (the templates' machine; a board image + // from a file carries its own machine's words). + if cfg.board == hw::Board::Templates { + for (offset, value) in [ + (0x2c, 8), + (0x30, 8), + (0x44, 800), + (0x7d8, 800), + (0x7e0, 800), + (0x81c, 800), + ] { + put32(&mut hw_img, hwdata::HWDATA_A + offset, value)?; + } } hwdata_obj.write(0, &hw_img)?; globals_obj.write(0, &glob)?; @@ -238,7 +368,7 @@ pub(crate) fn build( // Root. let r = &root_obj; - r.w64(0x00, root::MAGIC)?; + r.w64(0x00, layout.root_magic)?; r.w64(root::REGION_A, va::REGION_A)?; r.w64(root::RUNTIME, va::RUNTIME)?; r.w64(root::GLOBALS, va::GLOBALS)?; @@ -260,7 +390,7 @@ pub(crate) fn build( // Runtime pointers. let rt = &runtime_obj; - rt.w64(runtime::HWDATA, va::HWDATA)?; + rt.w64(runtime::HWDATA, hwdata_va)?; rt.w64(runtime::PAIR, va::PAIR)?; rt.w64(runtime::PAIR2, va::PAIR)?; for i in 0..13u64 { @@ -288,17 +418,17 @@ pub(crate) fn build( rt.w64(runtime::SMALL, va::SMALL)?; rt.w64( runtime::TA_ACTIVITY, - va::HWDATA + hwdata::TA_ACTIVITY as u64, + hwdata_va + hwdata::TA_ACTIVITY as u64, )?; rt.w64( runtime::FS_ACTIVITY, - va::HWDATA + hwdata::FS_ACTIVITY as u64, + hwdata_va + hwdata::FS_ACTIVITY as u64, )?; rt.w64( runtime::COMPUTE_ACTIVITY, - va::HWDATA + hwdata::COMPUTE_ACTIVITY as u64, + hwdata_va + hwdata::COMPUTE_ACTIVITY as u64, )?; - rt.w64(runtime::FAULT_INFO, va::HWDATA + hwdata::FAULT_INFO as u64)?; + rt.w64(runtime::FAULT_INFO, hwdata_va + layout.hwdata_view as u64)?; rt.w64(runtime::C0, va::C0)?; rt.w64(runtime::C1, va::C1)?; rt.w64(runtime::C1_2, va::C1)?; @@ -306,7 +436,7 @@ pub(crate) fn build( rt.w64(runtime::C3_2, va::C3)?; rt.w32(runtime::UNK_2D0, 4)?; rt.w32(runtime::UNK_3B0, 0xff)?; - rt.w64(runtime::HWDATA_A, va::HWDATA + hwdata::HWDATA_A as u64)?; + rt.w64(runtime::HWDATA_A, hwdata_va + hwdata::HWDATA_A as u64)?; // Control and flags. let c = &control_obj; diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index aeda1644ada0c..7b7ff0388b1b4 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -97,6 +97,8 @@ pub(crate) const GPU_GENERATION: u32 = 15; /// Static configuration of a SoC with a G15 GPU. pub(crate) struct Config { + /// SoC and GPU name, for the log. + pub(crate) name: &'static str, /// SoC identifier reported to userspace. pub(crate) chip_id: u32, /// GPU variant reported to userspace. @@ -114,18 +116,42 @@ pub(crate) struct Config { pub(crate) hw_ids: &'static [hw::HwId], /// Qualified firmware builds. pub(crate) firmware: &'static [hw::FirmwareBuild], + /// Machine compatible this SoC's boards carry. + pub(crate) machine: &'static CStr, + /// Where regions, segments and performance states come from, and how the GPU is powered. + pub(crate) platform: hw::Platform, + /// Initdata placement and sizes that differ between firmware builds. + pub(crate) layout: &'static hw::Layout, + /// Board images. + pub(crate) board: hw::Board, + /// Compute register values that scale with the cluster count. + pub(crate) compute: hw::ComputeRegs, +} + +/// Core slots of a SoC: the width of its enabled-core mask. +impl Config { + pub(crate) const fn core_slots(&self) -> u32 { + self.num_clusters * self.num_cores_per_cluster + } } /// OF match table of the G15 platform driver. /// /// Deliberately not exported with `module_device_table!`: without a module alias, the asahi /// module is not loaded at boot for this GPU, and the firmware only starts when userspace loads -/// the module after the native display is up (the J514S bring-up order). -const OF_TABLE: kernel::device_id::IdArray = - kernel::device_id::IdArray::new([( - of::DeviceId::new(c_str!("apple,agx-t6030")), - &hw::t6030::CONFIG, - )]); +/// the module after the native display is up (the J514S bring-up order). A built-in driver +/// probes when the GPU node is populated. +const OF_TABLE: kernel::device_id::IdArray = + kernel::device_id::IdArray::new([ + ( + of::DeviceId::new(c_str!("apple,agx-t6030")), + &hw::t6030::CONFIG, + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6031")), + &hw::t6031::CONFIG, + ), + ]); /// The G15 platform driver: the shared [`AsahiDriver`] probed through /// [`AsahiDriver::probe_g15`]. @@ -149,7 +175,7 @@ impl platform::Driver for Driver { info: Option<&Self::IdInfo>, ) -> impl PinInit { let cfg = *info.ok_or(ENODEV)?; - dev_info!(pdev.as_ref(), "G15: probing\n"); + dev_info!(pdev.as_ref(), "G15: probing ({})\n", cfg.name); match AsahiDriver::probe_g15(pdev, cfg) { Ok(driver) => Ok(Driver(Some(driver))), Err(error) if boot::glue::retained() => { @@ -290,6 +316,8 @@ impl Shared { /// be stopped, so dropping this structure leaks the firmware-visible parts on purpose /// ([`Gpu::retain`]). pub(crate) struct Firmware { + /// The SoC configuration. + pub(crate) cfg: &'static Config, /// RTKit, MMIO, regions, the adopted UAT and the host window. pub(crate) session: boot::Session, /// The initdata graph and its live words (Globals, Control, HardwareData, ...). @@ -362,7 +390,7 @@ impl Gpu { let uat = booted.session.uat()?; let (id, max_frequency_khz) = (booted.id, booted.max_frequency_khz); // The backend: graphs, slots, pools, engine, recovery and VM registry. - let firmware = Firmware::start_backend(dev, booted, &shared)?; + let firmware = Firmware::start_backend(dev, cfg, booted, &shared)?; *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { @@ -398,6 +426,7 @@ impl Firmware { /// retained as an orphan of `shared`. fn start_backend( dev: &AsahiDevice, + cfg: &'static Config, booted: boot::Booted, shared: &Arc, ) -> Result> { @@ -448,6 +477,7 @@ impl Firmware { let mut firmware = KBox::write( slot, Firmware { + cfg, session, init, objects, diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index e7993d1450919..112ed94bcaa55 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -36,9 +36,9 @@ pub(crate) fn resnap_us() -> u32 { (*module_parameters::g15_resnap_us.value()).clamp(25, 1000) } -/// Firmware idle standby time, us. -pub(crate) fn idle_standby_us() -> u32 { - (*module_parameters::g15_idle_standby_us.value()).clamp(100, 100000) +/// The explicit firmware idle standby time, us (0: not set). +pub(crate) fn idle_standby_override_us() -> u32 { + *module_parameters::g15_idle_standby_us.value() } /// Firmware idle minimum confidence, permille. @@ -56,6 +56,37 @@ pub(crate) fn stream_gate() -> bool { *module_parameters::g15_stream_gate.value() != 0 } +/// The 8-byte TEXT digest prefix of an unpinned firmware build to admit (0: none). +pub(crate) fn accept_text_prefix() -> u64 { + *module_parameters::g15_accept_text_prefix.value() +} + +/// The PB growth maximum the firmware is given, in blocks (64..=4095; the heap range holds +/// 4095 and the blocks above this answer the synchronous requests of partial renders). +pub(crate) fn pb_max_blocks() -> u32 { + (*module_parameters::g15_pb_max_blocks.value()).clamp(64, super::hw::tuning::PB_MAX_BLOCKS) +} + +/// The initial PB posting, in blocks (64..=4095; the ledger clamps it again). +pub(crate) fn pb_blocks() -> u32 { + (*module_parameters::g15_pb_blocks.value()).clamp(64, super::hw::tuning::PB_MAX_BLOCKS) +} + +/// PB reclaim at idle: 0 never, 1 re-initialize, 2 re-initialize and free the backing. +pub(crate) fn pb_reclaim() -> u32 { + (*module_parameters::g15_pb_reclaim.value()).min(2) +} + +/// Idle time before a PB reclaim, ns. +pub(crate) fn pb_idle_ns() -> u64 { + (*module_parameters::g15_pb_idle_ms.value()).max(100) as u64 * 1_000_000 +} + +/// Posted blocks above which an idle engine reclaims the PB. +pub(crate) fn pb_reclaim_blocks() -> u32 { + *module_parameters::g15_pb_reclaim_blocks.value() +} + /// Whether FRAGMENT_BARRIERS is advertised and mapped to early tiling. pub(crate) fn early_tiling() -> bool { *module_parameters::g15_early_tiling.value() != 0 @@ -79,7 +110,13 @@ pub(crate) fn fill(gpu: &Gpu, params: &mut uapi::drm_asahi_params_global) -> Res params.usc_generation = 3; params.gpu_hal_generation = uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY; params.max_attachments = hw::sizes::MAX_ATTACHMENTS as u32; - for (i, mask) in gpu.id.core_masks.iter().enumerate() { + for (i, mask) in gpu + .id + .core_masks + .iter() + .enumerate() + .take(gpu.cfg.num_clusters as usize) + { *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); } Ok(()) diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 6bb63c89f2ea7..36f27b3aa5856 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -63,6 +63,15 @@ fn stats_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { } } +/// The `asahi-g15/pstate` file contents. +fn pstate_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let state = shared.state.lock(); + match state.as_deref() { + Some(firmware) => firmware.write_pstate(f), + None => Ok(()), + } +} + impl Shared { /// One poll pass (the poll work item). pub(crate) fn run_poll(self: Arc) { @@ -151,17 +160,23 @@ impl Shared { self.mod_poll(1); } - /// Starts what the engine needs besides the firmware state: the `asahi-g15/stats` file. + /// Starts what the engine needs besides the firmware state: the `asahi-g15/stats` and + /// `asahi-g15/pstate` files. pub(crate) fn start_engine(self: &Arc) -> Result { let dir = debugfs::Dir::new(c_str!("asahi-g15")); let file = KBox::pin_init( dir.read_callback_file(c_str!("stats"), self.clone(), &stats_show), GFP_KERNEL, )?; + let pstate = KBox::pin_init( + dir.read_callback_file(c_str!("pstate"), self.clone(), &pstate_show), + GFP_KERNEL, + )?; let mut state = self.state.lock(); let firmware = state.as_deref_mut().ok_or(ENODEV)?; firmware.engine.shared = Some(self.clone()); firmware.engine.debugfs = Some(file); + firmware.engine.debugfs_pstate = Some(pstate); Ok(()) } } @@ -192,6 +207,9 @@ impl Firmware { // prefixes published the retirement loop below drains first. if self.engine.flights.is_empty() { self.drain_rings()?; + // Idle bookkeeping of the PB: when the idle began, and the reclaim once it is due. + self.note_idle(); + self.reclaim_check(); if self.recovery_idle() { return Ok(false); } @@ -213,6 +231,9 @@ impl Firmware { self.engine.prof.polls += 1; self.engine.prof.poll_ns += now_ns().wrapping_sub(start); } + // The pass that retired the last flight is not re-armed: note the idle here, not only + // at the start of a pass that finds the flights empty. + self.note_idle(); published?; Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen) } diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index d6def418a9293..41f2d2cc1f933 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -5,10 +5,28 @@ //! Record layouts: [`super::fw::buffer`]; the ledger and request checks: [`growth`] //! (host-tested). //! -//! - One 256 MiB PB heap at [`super::hw::client::PB_HEAP`] (2048 blocks of 128 KiB); initial -//! [`super::hw::tuning::PB_BLOCKS`] (1024), maximum 2048; truthful K = 4096 in every profile -//! (PB reconstruction, also in recovery, synchronizes 8 * K bytes of the block list: K = -//! 0x26cd overran the 0x8000-byte list and crashed the second recovery). +//! - One PB heap range at [`super::hw::client::PB_HEAP`] (4096 blocks of 128 KiB of client VA, +//! GPU-only), **backed on demand** in chunks of [`growth::CHUNK_BLOCKS`] blocks (16 MiB): each +//! chunk is its own GEM object mapped at its fixed GVA in every user VM that can tile into it +//! (at VM creation; at refill into the VMs of the flights; at publication into the job's VM: +//! [`Firmware::map_heap`]) and nowhere in the firmware window (the firmware learns pages from +//! the page and block lists only). Initial `g15_pb_blocks` (64) blocks posted, +//! `Ledger::chunks_for` of them backed at boot (the posting and one reserve chunk); truthful +//! K = 4096 in every profile (PB reconstruction, also in recovery, synchronizes 8 * K bytes of +//! the block list: K = 0x26cd overran the 0x8000-byte list and crashed the second recovery). +//! - Backing: a request is granted from the backed blocks above the posted count; after the +//! reply, when fewer than a reserve chunk remain, [`Firmware::refill_backing`] allocates the +//! next chunk (the TA runs on meanwhile). Only a reserve that ran dry makes a request wait: it +//! stays queued, the chunk is backed in the same pass and the next pass answers it. +//! - Reclaim ([`Firmware::reclaim_check`], `g15_pb_reclaim`): posted blocks never come back +//! while a PB lives (the firmware recycles pages inside its own pool; the host has no view of +//! it), so an engine idle for `g15_pb_idle_ms` with more than `g15_pb_reclaim_blocks` posted +//! re-initializes the PB at its next render through the recovery re-init path (fresh lists +//! and manager, InitBM of the initial blocks, no halt) and frees the backing above the kept +//! chunks (also with fewer posted, when more is backed than they need); an engine idle for +//! 50 ms (a client boundary) with more than half the maximum posted re-initializes it the +//! same way but keeps the backing. The native driver does the equivalent by replacing a +//! live queue's PB manager. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -39,12 +57,14 @@ use kernel::{ use super::fw::buffer::{self, block_ctl, counter, misc, pb_manager, uma_manager, uma_pools}; use super::fw::channels::DevCtl; use super::fw::initdata::control; -use super::hw::{client, pte, sizes, tuning}; +use super::hw::{self, client, pte, sizes, tuning}; use super::object::{Class, FwObject, Objects, Placement, Request}; +use super::vm::VmState; use super::Firmware; use crate::driver::AsahiDevice; +use crate::gem; -pub(crate) use growth::Ledger; +pub(crate) use growth::{Answer, IdleClock, Ledger}; static_assert!(buffer::MAX_BLOCKS == sizes::PB_BLOCKS); static_assert!(buffer::MAX_PAGES == sizes::PB_PAGES); @@ -54,6 +74,46 @@ static_assert!(buffer::MAX_BLOCKS as u64 * buffer::BLOCK_BYTES == client::PB_HEA static_assert!(buffer::PAGE_LIST_SIZE as u64 == client::PB_PAGE_LIST_SIZE); static_assert!(uma_manager::SEEDS == sizes::FREELIST_SEED); static_assert!(uma_pools::POOL_SIZE == sizes::POOL_SIZE); +static_assert!(growth::MAX_CHUNKS * growth::CHUNK_BLOCKS == buffer::MAX_BLOCKS); +static_assert!(CHUNK_BYTES % hw::PAGE == 0); + +/// Bytes of one backing chunk. +pub(crate) const CHUNK_BYTES: u64 = growth::CHUNK_BLOCKS as u64 * buffer::BLOCK_BYTES; + +/// Client GVA of backing chunk `i`. +pub(crate) const fn chunk_gva(i: usize) -> u64 { + client::PB_HEAP + i as u64 * CHUNK_BYTES +} + +/// One backing chunk of the heap: a GPU-only GEM object (never CPU-mapped, never in the +/// firmware window), mapped into user VMs by [`VmState::map_heap`]. +pub(crate) struct Chunk { + backing: gem::ObjectRef, +} + +impl Chunk { + /// Allocates a chunk: zeroed pages, pinned and DMA-mapped, each one the UAT can name. + fn new(dev: &AsahiDevice) -> Result { + let backing = gem::new_kernel_object_wc(dev, CHUNK_BYTES as usize)?; + let sgt = gem::held_sg_table(&backing.gem)?; + for range in sgt.iter() { + let pa = range.dma_address() as u64; + let len = range.dma_len() as u64; + if pa == 0 + || (pa | len) & (hw::PAGE - 1) != 0 + || (pa + len - 1) & !(hw::PA_MASK | (hw::PAGE - 1)) != 0 + { + return Err(EPERM); + } + } + Ok(Self { backing }) + } + + /// The GEM object, for the VM mappings. + pub(crate) fn gem(&self) -> ARef { + self.backing.gem.clone() + } +} /// The allocator state a rebuild after an abort moves to the spare. struct AllocatorSet { @@ -97,12 +157,23 @@ pub(crate) struct PbStats { pub(crate) grown: u64, pub(crate) refused: u64, pub(crate) bad: u64, + /// Requests that waited a pass for backing. + pub(crate) waits: u64, + /// Chunks backed after boot. + pub(crate) refills: u64, + /// Idle re-initializations, and the chunks they freed. + pub(crate) reclaims: u64, + pub(crate) freed: u64, } /// Render allocator and PB state: the allocators, the PB ledger and the spare set. pub(crate) struct Pools { /// The GPU device, for logging (also used by recovery). pub(crate) dev: ARef, + /// The driver device, for chunk allocation. + adev: ARef, + /// The heap's backing chunks, from the heap base. + heap: KVec, uma: [UmaEngine; 2], /// The allocator set in use. live: AllocatorSet, @@ -113,11 +184,14 @@ pub(crate) struct Pools { spare_ready: bool, pb_pages: Arc, pb_blocks: Arc, - pb_heap: Arc, /// The PB growth ledger since the last (re)initialization. pub(crate) ledger: Ledger, /// C1 names the current initial pages. c1_written: bool, + /// The unanswered-at-the-maximum warning was given since the last initialization. + exhausted_logged: bool, + /// The engine's idle clock (nothing published, nothing pending), for the reclaim. + idle: IdleClock, pub(crate) stats: PbStats, } @@ -243,7 +317,8 @@ fn gva(object: &FwObject) -> Result { impl Pools { /// Allocates the render allocator objects: per engine the UMA pools, freelist, pool table, /// manager and counter; block control, PB counter, misc, scene list and PB manager; the - /// uncached spare set; the PB page list, block list and heap. Contents are written by + /// uncached spare set; the PB page list and block list; the heap's first chunks (the + /// initial posting and one reserve chunk). Contents are written by /// [`Firmware::init_render_pools`]. pub(crate) fn new(dev: &AsahiDevice, objects: &mut Objects) -> Result { let (uma0, manager0, counter0) = UmaEngine::alloc(objects, 0)?; @@ -305,29 +380,76 @@ impl Pools { 0, true, )?; - let pb_heap = alloc( - objects, - client::PB_HEAP_SIZE, - Class::Shared, - Some((client::PB_HEAP, pte::LEAF_CACHED)), - 0, - false, - )?; + let mut ledger = Ledger::new( + tuning::PB_GROW, + super::params::pb_blocks(), + tuning::PB_MAX_BLOCKS, + ); + let chunks = Ledger::chunks_for(ledger.initial) as usize; + let mut heap = KVec::with_capacity(growth::MAX_CHUNKS as usize, GFP_KERNEL)?; + for _ in 0..chunks { + heap.push(Chunk::new(dev)?, GFP_KERNEL)?; + ledger.back_chunk(); + } + dev_info!( + dev.as_ref(), + "G15: PB heap: {} blocks posted at start, {} chunks of {} MiB backed\n", + ledger.initial, + chunks, + CHUNK_BYTES >> 20 + ); Ok(Self { dev: dev.as_ref().into(), + adev: dev.into(), + heap, uma: [uma0, uma1], live, spare_ready: spare.is_some(), spare, pb_pages, pb_blocks, - pb_heap, - ledger: Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS), + ledger, c1_written: false, + exhausted_logged: false, + idle: IdleClock::default(), stats: PbStats::default(), }) } + /// Frees the backing chunks from index `keep` on (their VM mappings are dropped by the + /// caller); the RAM returns when the last mapping of a chunk is gone. + fn free_chunks(&mut self, keep: usize) -> usize { + let freed = self.heap.len().saturating_sub(keep); + self.heap.truncate(keep); + self.ledger.unback_to(self.heap.len() as u32); + freed + } + + /// The GEM objects of the backed chunks, in heap order (for a new VM's mappings). + pub(crate) fn heap_gems(&self) -> Result>> { + let mut gems = KVec::with_capacity(self.heap.len(), GFP_KERNEL)?; + for chunk in self.heap.iter() { + gems.push(chunk.gem(), GFP_KERNEL)?; + } + Ok(gems) + } + + /// Maps the backed chunks `vm` does not have yet. + pub(crate) fn map_heap(&self, vm: &VmState) -> Result { + vm.map_heap(self.heap.iter().map(Chunk::gem)) + } + + /// Backs one more chunk; the caller maps it where needed before posting from it. + fn back_chunk(&mut self) -> Result> { + if self.heap.len() >= growth::MAX_CHUNKS as usize { + return Err(ENOSPC); + } + let chunk = Chunk::new(&self.adev)?; + let gem = chunk.gem(); + self.heap.push(chunk, GFP_KERNEL)?; + Ok(gem) + } + /// First rebuild after an abort with cached managers: the cached set may still have dirty /// lines in the ASC, so it is retained untouched and the graph moves to the never-exposed /// uncached spare (and its own counters and scene list), which [`Self::init`] then @@ -379,10 +501,10 @@ impl Pools { count.w32(0, 1)?; } - // PB: 8192 pages of 0x8000 in 2048 blocks of 0x20000; with growth the first initial - // blocks of them and page limits at the maximum. In every profile the block list - // declares its real 4096-record capacity. - self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + // PB: the initial blocks of the heap range posted, page limits at the maximum, the + // backing kept. In every profile the block list declares its real 4096-record capacity. + self.ledger.reinit(); + self.exhausted_logged = false; if tuning::PB_GROW { // C1 names the initial pages again. self.c1_written = false; @@ -397,7 +519,7 @@ impl Pools { self.pb_pages.fill(0, 0, buffer::PAGE_LIST_SIZE)?; self.pb_blocks.fill(0, 0, buffer::BLOCK_LIST_SIZE)?; } - let heap = gva(&self.pb_heap)?; + let heap = client::PB_HEAP; let initial = self.ledger.initial; write_records( &self.pb_pages, @@ -411,9 +533,13 @@ impl Pools { })?; set.pb_manager.write( 0, + // The maximum the firmware sees is the asynchronous growth limit, below the mapped + // heap: the blocks above it answer the synchronous requests a partial render makes + // (the firmware cannot finish a scene at its maximum without them; refused, its TA + // stalls and no kill ends it: ticket malik-na/linux-aurora#69). &buffer::pb_manager_image( initial, - self.ledger.max, + self.soft_max(), gva(&self.pb_pages)?, gva(&self.pb_blocks)?, set.block_ctl.va, @@ -440,18 +566,32 @@ impl Pools { } } - /// The debugfs line `pb: blocks initial/posted/max requests grown refused bad`. + /// The firmware's growth maximum in blocks (`g15_pb_max_blocks`), at least the initial size + /// and at most the mapped heap. + pub(crate) fn soft_max(&self) -> u32 { + super::params::pb_max_blocks().clamp(self.ledger.initial, self.ledger.max) + } + + /// The debugfs line `pb: blocks initial/posted/firmware-max/max requests grown refused bad + /// backed chunks waits refills` (the first eight keys are stable; new ones are appended). pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { writeln!( f, - "pb: blocks {}/{}/{} requests {} grown {} refused {} bad {}", + "pb: blocks {}/{}/{}/{} requests {} grown {} refused {} bad {} backed {} chunks {} waits {} refills {} reclaims {} freed {}", self.ledger.initial, self.ledger.posted, + self.soft_max(), self.ledger.max, self.stats.requests, self.stats.grown, self.stats.refused, - self.stats.bad + self.stats.bad, + self.ledger.backed, + self.heap.len(), + self.stats.waits, + self.stats.refills, + self.stats.reclaims, + self.stats.freed ) } } @@ -482,35 +622,178 @@ impl Firmware { /// Answers the firmware's PB-growth requests, outside the event drain. /// A request names PB slot 0, the context of a published render prefix and the next /// request counter; anything else is logged and left unanswered (its TA then stays stalled - /// and the deadline applies). With growth up to ten more blocks of the mapped heap are + /// and the deadline applies). With growth up to ten more backed blocks of the heap are /// written to unposted block-list records, then the live block_ctl+4 producer advances and /// DEVCTL 7 reports success; at the maximum the reply is the refusal (+4 = 0), which lets /// the firmware continue without new pages. The firmware imports the blocks itself: P is - /// never rewritten. A failed reply is logged; a request that cannot be processed at all (a + /// never rewritten. A request with no backed block left waits: it stays at the head of the + /// queue for the next pass while this one backs a chunk. After the answers, the reserve is + /// refilled. A failed reply is logged; a request that cannot be processed at all (a /// firmware object access failed) is logged and left unanswered like an invalid one. Never /// fails: the poll pass continues. pub(crate) fn answer_growth(&mut self) -> Result { - while !self.events.grow.is_empty() { + // Only the requests queued before this pass: a reply's wait drains the rings and may + // queue the next request, which the next pass answers (the device lock is released in + // between). + let queued = self.events.grow.len(); + for _ in 0..queued { let Ok(request) = self.events.grow.remove(0) else { break; }; - self.pools.stats.requests += 1; - if let Err(e) = self.answer_request(&request) { - self.pools.stats.bad += 1; - dev_err!( + match self.answer_request(&request) { + Ok(true) => self.pools.stats.requests += 1, + Ok(false) => { + // Waits for backing: back at the head, counted once answered. + self.pools.stats.waits += 1; + // The queue has room: this request was just taken from it. + if self.events.grow.insert_within_capacity(0, request).is_err() { + self.pools.stats.bad += 1; + } + break; + } + Err(e) => { + self.pools.stats.requests += 1; + self.pools.stats.bad += 1; + dev_err!( + self.dev(), + "PB growth request {} ctx {} not answered: {}\n", + request[buffer::grow_request::COUNTER], + request[buffer::grow_request::CONTEXT], + e.to_errno() + ); + } + } + } + self.refill_backing(); + Ok(()) + } + + /// Backs chunks until the reserve is whole, mapping each into the VMs of the flights before + /// the ledger may post from it (a job's VM is mapped at publication, a new VM at creation). + /// A failure is logged and leaves the reserve short: requests then wait and the next pass + /// tries again. + pub(crate) fn refill_backing(&mut self) { + // A pending re-initialization is about to drop the posted count: nothing to back for. + if self.recovery.reinit { + return; + } + while self.pools.ledger.refill_due() { + let result = (|| -> Result { + let gem = self.pools.back_chunk()?; + let i = self.pools.heap.len() - 1; + for flight in self.engine.flights.iter() { + flight.job.vm.map_chunk(i, gem.clone())?; + } + Ok(()) + })(); + match result { + Ok(()) => { + self.pools.ledger.back_chunk(); + self.pools.stats.refills += 1; + dev_dbg!( + self.dev(), + "PB heap: chunk {} backed ({} of {} blocks posted)\n", + self.pools.heap.len() - 1, + self.pools.ledger.posted, + self.pools.ledger.backed + ); + } + Err(e) => { + dev_warn!( + self.dev(), + "PB heap: backing chunk {} failed: {} ({} of {} blocks posted)\n", + self.pools.heap.len(), + e.to_errno(), + self.pools.ledger.posted, + self.pools.ledger.backed + ); + return; + } + } + } + } + + /// Maps the backed heap chunks into a job's VM before its publication. + pub(crate) fn map_heap(&self, vm: &VmState) -> Result { + self.pools.map_heap(vm) + } + + /// Notes the engine idle (nothing published, nothing pending) or busy; the idle time is + /// what [`Self::reclaim_check`] measures. Called at the end of every poll pass and + /// publication (the pass that retires the last flight is not re-armed) and at a job's + /// arrival. + pub(crate) fn note_idle(&mut self) { + let idle = self.engine.flights.is_empty() && self.engine.pending.is_empty(); + self.pools.idle.note(idle, super::engine::now_ns()); + } + + /// Reclaims the PB when the engine has been idle for `g15_pb_idle_ms` with more than + /// `g15_pb_reclaim_blocks` posted or more backing than the posted blocks need, or for + /// [`growth::DRAIN_IDLE_NS`] with more than half the maximum posted + /// ([`growth::Ledger::drain_reinit_due`]; re-initialization only): the next render prefix + /// re-initializes the render allocators and the PB (InitBM of the initial blocks; + /// `recovery.reinit`, as after a recovered abort but with no halt), and with + /// `g15_pb_reclaim` 2 the backing chunks above those of the initial posting and the reserve + /// are freed now, unmapped from every open VM + /// (a VM that publishes later drops its stale mappings at [`VmState::map_heap`]). Called + /// under the device lock when a job arrives at an idle engine and in idle poll passes. + pub(crate) fn reclaim_check(&mut self) { + let level = super::params::pb_reclaim(); + if level == 0 + || !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || self.recovery.reinit + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + let floor = super::params::pb_reclaim_blocks(); + let min_idle = super::params::pb_idle_ns(); + let ledger = &self.pools.ledger; + let idle = ledger.reclaim_due(floor, idle_ns, min_idle) + || (level >= 2 && ledger.free_due(idle_ns, min_idle)); + if !idle && !self.pools.ledger.drain_reinit_due(idle_ns) { + return; + } + self.recovery.reinit = true; + self.pools.stats.reclaims += 1; + let posted = self.pools.ledger.posted; + let mut freed = 0; + // A short idle keeps the backing: the next scene would allocate it again. + if level >= 2 && idle { + let keep = self.pools.ledger.keep_chunks() as usize; + freed = self.pools.free_chunks(keep); + self.pools.stats.freed += freed as u64; + // The mappings hold the chunks' pages: drop them in every VM that could publish. + match self.vms.open() { + Ok(states) => { + for state in states.iter() { + state.unmap_heap_above(keep); + } + } + Err(e) => dev_warn!( self.dev(), - "PB growth request {} ctx {} not answered: {}\n", - request[buffer::grow_request::COUNTER], - request[buffer::grow_request::CONTEXT], + "PB heap: reclaim could not walk the VMs: {}\n", e.to_errno() - ); + ), } } - Ok(()) + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted: re-initialization to {} at the next render, {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + self.pools.ledger.initial, + freed, + self.pools.heap.len() + ); } - /// Processes one growth request. - fn answer_request(&mut self, request: &[u32; 14]) -> Result { + /// Processes one growth request: Ok(false) when it waits for backing. + fn answer_request(&mut self, request: &[u32; 14]) -> Result { let ctl = self.init.control.clone().ok_or(ENODEV)?; let epoch = ctl.r32(control::HALT_COUNT)? as u64 | (ctl.r32(control::HALT_COUNT + 4)? as u64) << 32; @@ -521,6 +804,23 @@ impl Firmware { .iter() .enumerate() .any(|(i, f)| self.receipts(i).renders != 0 && f.job.vm.slot == ctx); + // At the maximum, after the refusals, the request is left to the deadline: a scene the + // firmware cannot finish without more PB stalls its TA, and the host deadline recovers. + if self.pools.ledger.exhausted() { + self.pools.stats.bad += 1; + if !self.pools.exhausted_logged { + self.pools.exhausted_logged = true; + dev_warn!( + self.dev(), + "PB growth request {} ctx {} left unanswered: {} refusals at the maximum ({} blocks); the host deadline applies\n", + request[buffer::grow_request::COUNTER], + ctx, + self.pools.ledger.refusals, + self.pools.ledger.max + ); + } + return Ok(true); + } // A request from before a halt has no authority over the PB now. if !self .pools @@ -540,10 +840,14 @@ impl Firmware { live as u32, tuning::PB_GROW as u32 ); - return Ok(()); + return Ok(true); } - let (first, n) = self.pools.ledger.accept(request); - let heap = gva(&self.pools.pb_heap)?; + let (first, n) = match self.pools.ledger.accept(request) { + Answer::Grant { first, count } => (first, count), + Answer::Refuse => (self.pools.ledger.posted, 0), + Answer::Wait => return Ok(false), + }; + let heap = client::PB_HEAP; for b in first..first + n { self.pools.pb_blocks.w64( buffer::block_record_offset(b), @@ -569,7 +873,11 @@ impl Firmware { }; let ret = self.devctl(&reply, true); let ledger = self.pools.ledger; - let info = self.pools.ledger.log_answer(request, n, ret.is_err()); + let soft = self.pools.soft_max(); + // Every answer past the firmware's maximum is logged with its TA position: a scene + // that keeps asking at one position is a runaway, one that advances is large. + let past_soft = n != 0 && ledger.posted > soft; + let info = self.pools.ledger.log_answer(request, n, ret.is_err()) || past_soft; let (counter, position) = ( request[buffer::grow_request::COUNTER], request[buffer::grow_request::POSITION], @@ -603,6 +911,6 @@ impl Firmware { ret ); } - Ok(()) + Ok(true) } } diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 778d19ef23b55..3df2303e69893 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -1,18 +1,51 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! The PB growth ledger and the growth-request checks. +//! The PB growth ledger, its backing account and the growth-request checks. //! //! The firmware asks for more PB with an event of type 7 naming PB slot 0, the context of a //! published render, its request counter (P+0x60 before its increment), the TA position and the -//! halt epoch; the host appends at most ten blocks of the mapped heap to the block list and -//! advances the block_ctl+4 producer (modulo K), then replies with DEVCTL 7. At the maximum the -//! reply refuses, and the firmware finishes the scene with partial renders. A request from -//! before a halt (epoch) or not matching a published render prefix is left unanswered. +//! halt epoch; the host appends at most ten blocks of the heap to the block list and advances +//! the block_ctl+4 producer (modulo K), then replies with DEVCTL 7. At the maximum the reply +//! refuses, and the firmware finishes the scene with partial renders. A request from before a +//! halt (epoch) or not matching a published render prefix is left unanswered. +//! +//! The heap's VA range covers [`MAX_BLOCKS`] blocks but its RAM comes in chunks of +//! [`CHUNK_BLOCKS`] blocks, backed on demand from the heap base: the ledger counts them as +//! `backed`. A request is granted from the backed blocks above `posted`; a chunk is due +//! ([`Ledger::refill_due`]) whenever fewer than [`RESERVE_BLOCKS`] backed blocks remain above +//! the posted count, so the reply path itself never waits for an allocation; only if the +//! reserve ran dry does a request wait ([`Answer::Wait`]) for the chunk the same pass backs. A +//! re-initialization ([`Ledger::reinit`]) returns the posted count to the initial size and +//! keeps the backing; [`Ledger::keep_chunks`] says how much of it the initial posting and the +//! reserve need. //! //! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. use crate::g15::fw::buffer::{grow_request, BLOCK_LIST_RECORDS, GROW_STEP, MAX_BLOCKS, MIN_BLOCKS}; +/// Blocks per backing chunk (16 MiB). +pub(crate) const CHUNK_BLOCKS: u32 = 128; +/// Backed blocks kept above the posted count: at 1-2 ms per request of ten blocks, 13 requests +/// of lead for the allocation of the next chunk. +pub(crate) const RESERVE_BLOCKS: u32 = CHUNK_BLOCKS; +/// Chunks of the whole heap. +/// The idle time before a re-initialization past half the maximum (a client boundary, not a +/// gap between one client's renders). +pub(crate) const DRAIN_IDLE_NS: u64 = 50_000_000; + +pub(crate) const MAX_CHUNKS: u32 = MAX_BLOCKS / CHUNK_BLOCKS; + +/// The answer to an admitted growth request. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Answer { + /// Post blocks `[first, first + count)` (`count` at most ten) and reply grown. + Grant { first: u32, count: u32 }, + /// Reply refused: the maximum is posted. + Refuse, + /// No backed block above the posted count: leave the request queued and back a chunk. + Wait, +} + /// The PB ledger since the last (re)initialization. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) struct Ledger { @@ -22,53 +55,154 @@ pub(crate) struct Ledger { pub(crate) max: u32, /// Blocks posted to the block list, cumulative (the block_ctl+4 producer is this modulo K). pub(crate) posted: u32, + /// Blocks with RAM behind them, from the heap base: whole chunks. + pub(crate) backed: u32, /// The next expected request counter. pub(crate) next_request: u32, /// A refusal at the maximum was logged since the last initialization. pub(crate) refusal_logged: bool, + /// Refusals answered at the maximum since the last initialization. After + /// [`MAX_REFUSALS`] further requests are left unanswered: a firmware that keeps asking at + /// the maximum instead of finishing the scene with partial renders would otherwise be + /// answered forever, under the device lock (ticket malik-na/linux-aurora#69); unanswered, + /// its TA stalls and the host deadline recovers. + pub(crate) refusals: u32, } +/// The most blocks ever posted: one below the block list's records, as the producer is the +/// posted count modulo the list and a whole list would read as an empty one (0). +pub(crate) const POSTED_LIMIT: u32 = min(MAX_BLOCKS, BLOCK_LIST_RECORDS - 1); + +/// Refusals answered at the maximum before requests are left to the deadline. +pub(crate) const MAX_REFUSALS: u32 = 2; + impl Ledger { - /// A fresh ledger. With growth, the initial size is clamped to 64..=2048 blocks and the - /// maximum to initial..=2048; without, both are the fixed 2048-block PB. + /// A fresh ledger. With growth, the initial size is clamped to 64..=[`POSTED_LIMIT`] blocks + /// and the maximum to initial..=[`POSTED_LIMIT`]; without, both are [`POSTED_LIMIT`]. pub(crate) const fn new(grow: bool, blocks: u32, max_blocks: u32) -> Self { let (initial, max) = if grow { - let nb = clamp(blocks, MIN_BLOCKS, MAX_BLOCKS); - (nb, clamp(max_blocks, nb, MAX_BLOCKS)) + let nb = clamp(blocks, MIN_BLOCKS, POSTED_LIMIT); + (nb, clamp(max_blocks, nb, POSTED_LIMIT)) } else { - (MAX_BLOCKS, MAX_BLOCKS) + (POSTED_LIMIT, POSTED_LIMIT) }; Self { initial, max, posted: initial, + backed: 0, next_request: 0, refusal_logged: false, + refusals: 0, } } + /// A re-initialization: the initial blocks are posted again, the request counter starts + /// over, the backing stays. + pub(crate) const fn reinit(&mut self) { + self.posted = self.initial; + self.next_request = 0; + self.refusal_logged = false; + self.refusals = 0; + } + /// The live block_ctl+4 producer. pub(crate) const fn producer(&self) -> u32 { self.posted % BLOCK_LIST_RECORDS } + /// Records one more backed chunk. + pub(crate) const fn back_chunk(&mut self) { + self.backed += CHUNK_BLOCKS; + } + + /// Drops the backing to `chunks` chunks. + pub(crate) const fn unback_to(&mut self, chunks: u32) { + self.backed = chunks * CHUNK_BLOCKS; + } + + /// Backed blocks above the posted count. + pub(crate) const fn spare(&self) -> u32 { + self.backed.saturating_sub(self.posted) + } + + /// Whether another chunk is due: fewer than the reserve above the posted count and the heap + /// not yet backed whole. + pub(crate) const fn refill_due(&self) -> bool { + self.backed < MAX_BLOCKS && self.spare() < RESERVE_BLOCKS + } + + /// Chunks that `blocks` posted blocks and the reserve need. + pub(crate) const fn chunks_for(blocks: u32) -> u32 { + let n = (blocks + RESERVE_BLOCKS).div_ceil(CHUNK_BLOCKS); + if n > MAX_CHUNKS { + MAX_CHUNKS + } else { + n + } + } + + /// Chunks a re-initialized heap keeps: those of the initial posting and the reserve. + pub(crate) const fn keep_chunks(&self) -> u32 { + Self::chunks_for(self.initial) + } + + /// Whether the heap is to be re-initialized: idle for `idle_ns` with more than `floor` + /// blocks posted. + pub(crate) const fn reclaim_due(&self, floor: u32, idle_ns: u64, min_idle_ns: u64) -> bool { + self.posted > floor && idle_ns >= min_idle_ns + } + + /// Whether the backing is to be freed: idle for `idle_ns` with more chunks backed than the + /// posted blocks and the reserve need. A client-boundary re-initialization keeps the + /// backing of a heavy scene; if the next client posts below the reclaim floor, this frees it. + pub(crate) const fn free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS + } + + /// Whether a short idle re-initializes the heap: more than half the maximum posted and + /// idle for [`DRAIN_IDLE_NS`]. Back-to-back heavy scenes (a new client every few hundred + /// ms) never reach `g15_pb_idle_ms`, and posted blocks never come back while a PB lives, so + /// without this they run the heap to the maximum. A re-initialization with no idle time, + /// between two renders of one client, faulted at the heap base (r2c boot). + pub(crate) const fn drain_reinit_due(&self, idle_ns: u64) -> bool { + self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS + } + + /// Whether the refusals at the maximum are used up: further requests go unanswered. + pub(crate) const fn exhausted(&self) -> bool { + self.refusals >= MAX_REFUSALS + } + /// Whether `request` may be answered: growth on, an event of type 7 for PB slot 0 from a /// context with a published render prefix (`live`), a counter not below the next expected - /// one and the current halt epoch. + /// one, the current halt epoch, and refusals at the maximum not used up. pub(crate) fn admits(&self, request: &[u32; 14], grow: bool, live: bool, epoch: u64) -> bool { - grow && request[grow_request::TYPE] == 7 + !self.exhausted() + && grow + && request[grow_request::TYPE] == 7 && request[grow_request::SLOT] == 0 && live && request[grow_request::COUNTER] >= self.next_request && request_epoch(request) == epoch } - /// Accepts an admitted request: the next expected counter follows it, and the returned - /// blocks `[first, first + count)` are to be written to the block list, `count` 0 meaning a - /// refusal at the maximum. - pub(crate) fn accept(&mut self, request: &[u32; 14]) -> (u32, u32) { + /// Accepts an admitted request. Granted or refused, the next expected counter follows it; + /// a request that waits for backing is accepted again when the chunk is there. + pub(crate) fn accept(&mut self, request: &[u32; 14]) -> Answer { + let room = self.max - self.posted; + if room != 0 && self.spare() == 0 { + return Answer::Wait; + } self.next_request = request[grow_request::COUNTER].wrapping_add(1); - (self.posted, GROW_STEP.min(self.max - self.posted)) + if room == 0 { + self.refusals += 1; + return Answer::Refuse; + } + Answer::Grant { + first: self.posted, + count: GROW_STEP.min(room).min(self.spare()), + } } /// Records `count` blocks as posted after their records were written and returns the new @@ -94,11 +228,50 @@ impl Ledger { } } +/// The engine's idle clock for the reclaim policy: when nothing was published and nothing +/// pending, from the pass that retired the last flight on. Noted at the end of every poll pass +/// and publication (a pass that empties the flights is not re-armed, so a note only at the +/// start of a pass with empty flights depends on a backup pass happening to run) and, busy, +/// when a job is enqueued. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct IdleClock { + /// When the idle began; 0 while busy. + since_ns: u64, +} + +impl IdleClock { + /// Notes the engine idle or busy at `now_ns`; an idle already begun keeps its start. + pub(crate) fn note(&mut self, idle: bool, now_ns: u64) { + if !idle { + self.since_ns = 0; + } else if self.since_ns == 0 { + self.since_ns = now_ns.max(1); + } + } + + /// The idle time at `now_ns`; 0 while busy. + pub(crate) fn idle_ns(&self, now_ns: u64) -> u64 { + if self.since_ns == 0 { + 0 + } else { + now_ns.wrapping_sub(self.since_ns) + } + } +} + /// The halt epoch a request names (u64 at words 5..6). pub(crate) const fn request_epoch(request: &[u32; 14]) -> u64 { request[grow_request::EPOCH] as u64 | (request[grow_request::EPOCH + 1] as u64) << 32 } +const fn min(a: u32, b: u32) -> u32 { + if a < b { + a + } else { + b + } +} + const fn clamp(v: u32, lo: u32, hi: u32) -> u32 { if v < lo { lo @@ -125,26 +298,53 @@ mod tests { q } + /// A ledger with the heap backed whole. + fn backed(grow: bool, blocks: u32, max_blocks: u32) -> Ledger { + let mut l = Ledger::new(grow, blocks, max_blocks); + l.unback_to(MAX_CHUNKS); + l + } + #[test] fn clamps() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); assert_eq!( - (l.initial, l.max, l.posted, l.next_request), - (1024, 2048, 1024, 0) + (l.initial, l.max, l.posted, l.backed, l.next_request), + (1024, 2048, 1024, 4096, 0) ); assert_eq!(l.producer(), 1024); assert_eq!(Ledger::new(true, 10, 20).initial, 64); assert_eq!(Ledger::new(true, 10, 20).max, 64); - assert_eq!(Ledger::new(true, 4000, 100).initial, 2048); + assert_eq!(Ledger::new(true, 5000, 100).initial, POSTED_LIMIT); assert_eq!(Ledger::new(true, 512, 1024).max, 1024); assert_eq!(Ledger::new(true, 512, 100).max, 512); let fixed = Ledger::new(false, 512, 1024); - assert_eq!((fixed.initial, fixed.max, fixed.posted), (2048, 2048, 2048)); + assert_eq!( + (fixed.initial, fixed.max, fixed.posted), + (POSTED_LIMIT, POSTED_LIMIT, POSTED_LIMIT) + ); + } + + /// No construction posts a whole block list: the producer would wrap to 0, an empty list. + #[test] + fn producer_never_wraps() { + for (grow, blocks, max) in [ + (true, 4096, 4096), + (true, 5000, 9000), + (true, 64, 4096), + (false, 0, 0), + ] { + let mut l = backed(grow, blocks, max); + assert_ne!(l.producer(), 0); + l.posted = l.max; + assert_eq!(l.producer(), l.max); + assert!(l.max < BLOCK_LIST_RECORDS); + } } #[test] fn admission() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); let q = request(5, 0, 3); assert!(l.admits(&q, true, true, 3)); assert!(!l.admits(&q, false, true, 3)); @@ -166,17 +366,20 @@ mod tests { #[test] fn growth_to_the_maximum() { // 512 initial and 1024 maximum blocks. - let mut l = Ledger::new(true, 512, 1024); + let mut l = backed(true, 512, 1024); let mut grown = 0; let mut counter = 0; loop { let q = request(5, counter, 1); assert!(l.admits(&q, true, true, 1)); - let (first, n) = l.accept(&q); - assert_eq!(first, 512 + grown); - if n == 0 { - break; - } + let n = match l.accept(&q) { + Answer::Grant { first, count } => { + assert_eq!(first, 512 + grown); + count + } + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert!(n <= 10); assert_eq!(l.post(n), (512 + grown + n) % 4096); grown += n; @@ -189,17 +392,234 @@ mod tests { // A stale (repeated) counter is refused, a later one accepted. assert!(!l.admits(&request(5, counter - 1, 1), true, true, 1)); assert!(l.admits(&request(5, counter + 3, 1), true, true, 1)); + // One refusal was answered above; one more is, then requests go unanswered. + assert_eq!(l.refusals, 1); + assert!(!l.exhausted()); + let q = request(5, counter + 3, 1); + assert_eq!(l.accept(&q), Answer::Refuse); + assert!(l.exhausted()); + assert!(!l.admits(&request(5, counter + 4, 1), true, true, 1)); + // A fresh ledger starts over, and so does a re-initialized one (backing kept). + assert!(!Ledger::new(true, 512, 1024).exhausted()); + l.reinit(); + assert_eq!( + (l.posted, l.next_request, l.refusals, l.backed), + (512, 0, 0, 4096) + ); + assert!(!l.exhausted() && !l.refusal_logged); } #[test] fn producer_wraps_at_k() { - let mut l = Ledger::new(true, 2048, 2048); + let mut l = backed(true, 2048, 2048); assert_eq!(l.producer(), 2048); l.posted = 4095; assert_eq!(l.post(1), 0); assert_eq!(l.post(10), 10); } + #[test] + fn backing() { + // 64 initial blocks: two chunks at boot (the initial posting and the reserve). + assert_eq!(Ledger::chunks_for(64), 2); + assert_eq!(Ledger::chunks_for(1024), 9); + assert_eq!(Ledger::chunks_for(4095), MAX_CHUNKS); + assert_eq!(MAX_CHUNKS, 32); + let mut l = Ledger::new(true, 64, 4095); + assert_eq!(l.keep_chunks(), 2); + assert_eq!(l.spare(), 0); + assert!(l.refill_due()); + // Nothing backed: a request waits and the counter does not advance. + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + l.back_chunk(); + l.back_chunk(); + assert_eq!((l.backed, l.spare()), (256, 192)); + assert!(!l.refill_due()); + // Granting until the reserve line, then a refill is due; at the end of the backing the + // grant is clipped and the next request waits. + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 10 + } + ); + assert_eq!(l.post(10), 74); + assert_eq!(l.next_request, 1); + l.posted = 128; + assert!(!l.refill_due()); + l.posted = 129; + assert!(l.refill_due()); + l.posted = 250; + assert_eq!( + l.accept(&request(3, 1, 1)), + Answer::Grant { + first: 250, + count: 6 + } + ); + l.post(6); + assert_eq!(l.accept(&request(3, 2, 1)), Answer::Wait); + // The heap end: no refill past the whole heap, the maximum refuses. + l.unback_to(MAX_CHUNKS); + l.posted = 4090; + assert!(!l.refill_due()); + assert_eq!( + l.accept(&request(3, 2, 1)), + Answer::Grant { + first: 4090, + count: 5 + } + ); + l.post(5); + assert_eq!(l.accept(&request(3, 3, 1)), Answer::Refuse); + // Re-initialization keeps the backing; dropping it to the kept chunks. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 4096)); + l.unback_to(l.keep_chunks()); + assert_eq!(l.backed, 256); + } + + #[test] + fn reclaim() { + let mut l = Ledger::new(true, 64, 4095); + let s = 1_000_000_000; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1024; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1025; + assert!(l.reclaim_due(1024, s, s)); + assert!(!l.reclaim_due(1024, s - 1, s)); + // The clock: busy is 0, an idle keeps its start, busy ends it. + let mut c = IdleClock::default(); + assert_eq!(c.idle_ns(10 * s), 0); + c.note(true, 10 * s); + c.note(true, 12 * s); + assert_eq!(c.idle_ns(15 * s), 5 * s); + c.note(false, 16 * s); + assert_eq!(c.idle_ns(20 * s), 0); + // A clock started at 0 ns still counts as idle. + c.note(true, 0); + assert_eq!(c.idle_ns(s), s - 1); + } + + /// The r2b warm boot's sequence: heavy scenes of ~1420 blocks each, new clients back to + /// back. The engine re-initializes at a client boundary once past half the maximum, so the + /// heap never reaches it; the backing stays (no refill on the next scene). + #[test] + fn back_to_back_heavy_scenes() { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + let mut counter = 0; + for _ in 0..10 { + if l.drain_reinit_due(DRAIN_IDLE_NS) { + l.reinit(); + counter = l.next_request; + } + let start = l.posted; + while l.posted < start + 1420 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + assert!(l.posted < l.max); + } + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + assert_eq!(l.backed, 24 * CHUNK_BLOCKS); + l.posted = 2047; + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + l.posted = 2048; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // Not between two renders of one client. + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS - 1)); + assert!(!l.drain_reinit_due(0)); + } + + /// The regression boot's sequence: a client-boundary re-initialization keeps the backing + /// of a heavy scene, the next client posts little; once idle, the backing above what the + /// posted blocks need is due to be freed although the posted count is below the floor. + #[test] + fn kept_backing_freed_at_idle() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(!l.free_due(5 * s, s)); + l.reinit(); + l.posted = 264; + assert!(!l.reclaim_due(1024, 5 * s, s)); + assert!(!l.free_due(s - 1, s)); + assert!(l.free_due(s, s)); + // After the re-initialization and the free, nothing more is due. + l.reinit(); + l.unback_to(l.keep_chunks()); + assert!(!l.free_due(5 * s, s)); + // A heap backed for what it posts plus the reserve is not freed. + l.posted = 1000; + l.unback_to(Ledger::chunks_for(1000)); + assert!(!l.free_due(5 * s, s)); + } + + /// The r2 boot's sequence: a heavy scene, a reclaim that frees the backing, refills during + /// the next heavy scene, an idle gap with a small job in it, and the next render must + /// reclaim again. + #[test] + fn reclaim_after_free_then_refill() { + let s = 1_000_000_000; + let (floor, min_idle) = (1024, s); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + let mut clock = IdleClock::default(); + // A heavy scene grows the PB to 1484 blocks, refilling as it goes. + let mut heavy = |l: &mut Ledger, clock: &mut IdleClock, t: u64| { + clock.note(false, t); + let mut counter = l.next_request; + while l.posted < 1484 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + // The pass that retires the last flight notes the idle; no later pass may run. + clock.note(true, t + s); + }; + heavy(&mut l, &mut clock, 100 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // 5 s later a small job arrives: the reclaim is due; it frees the backing above the + // kept chunks; the re-initialization happens at the next render. + let t = 106 * s; + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + l.unback_to(l.keep_chunks()); + clock.note(false, t); + assert_eq!(l.backed, 2 * CHUNK_BLOCKS); + clock.note(true, t + s / 2); + // The next render re-initializes, then the second heavy scene refills to 1484 again. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 256)); + heavy(&mut l, &mut clock, 110 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // The same gap again: the reclaim must be due at the small job's arrival. + let t = 116 * s; + assert_eq!(clock.idle_ns(t), 5 * s); + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + } + #[test] fn logging() { let mut l = Ledger::new(true, 64, 64); diff --git a/drivers/gpu/drm/asahi/g15/power.rs b/drivers/gpu/drm/asahi/g15/power.rs index 1da3d5a99f754..58b6636fde383 100644 --- a/drivers/gpu/drm/asahi/g15/power.rs +++ b/drivers/gpu/drm/asahi/g15/power.rs @@ -5,23 +5,63 @@ //! The words and their rationale are in [`idle`]. Initdata writes them into the Globals image //! before the firmware first sees it: //! -//! - +0x7a8 standby = `g15_idle_standby_us` (2500, clamped 100..100000); +//! - +0x7a8 standby = `g15_idle_standby_us` when set (clamped 100..100000), else the GPU node's +//! `apple,idleoff-standby-timer`, else the tuned 2500 ([`idle::standby`]); //! - +0x7c4 minimum confidence = `g15_idle_min_conf` (500 permille as f32); //! - +0x30 predictive = 0 ([`super::hw::tuning::IDLE_PREDICTIVE`]): the finite delay, the GPU //! powers down after the +0x9a0 delay (template 2, about 2 ms); //! - +0x2c idle decision enabled (template 1, unchanged). //! -//! DVFS is firmware-run on the ADT tables. +//! DVFS is firmware-run on the ADT tables. The state it runs at is read back by +//! `asahi-g15/pstate` ([`pstate`], [`super::Firmware::write_pstate`]): the newest power-state +//! record of the stats ring, and HardwareDataA's state words. mod idle; -pub(crate) use idle::permille_f32; +pub(crate) mod pstate; +pub(crate) use idle::{permille_f32, standby}; -use kernel::prelude::*; +use kernel::{fmt, prelude::*}; -use super::fw::initdata::globals; +use super::board; +use super::fw::initdata::{globals, hwdata}; +use super::hw::rings; +use super::Firmware; static_assert!(idle::ENABLE == globals::IDLE_ENABLE); static_assert!(idle::PREDICTIVE == globals::IDLE_PREDICTIVE); static_assert!(idle::STANDBY_US == globals::IDLE_STANDBY_US); static_assert!(idle::MIN_CONF == globals::IDLE_MIN_CONF); static_assert!(idle::DELAY == globals::IDLE_DELAY); + +static_assert!(pstate::STATES == board::DVFS_STATES); +static_assert!(pstate::STATS_RECORD * pstate::STATS_SLOTS as usize == rings::STATS_RING_SIZE); +static_assert!(pstate::STATS_SLOTS == super::fw::channels::RING_SLOTS); + +impl Firmware { + /// HardwareDataA's state words, as the firmware last wrote them. + fn pstate_words(&self) -> Result { + let hw = self.init.hwdata.as_ref().ok_or(ENODEV)?; + let word = |offset: usize| hw.r32(hwdata::HWDATA_A + offset); + Ok(pstate::HwWords { + actual: word(pstate::HWDATA_A_ACTUAL)?, + target: word(pstate::HWDATA_A_TARGET)?, + current: word(pstate::HWDATA_A_CURRENT)?, + freq_mhz: word(pstate::HWDATA_A_FREQ_MHZ)?, + }) + } + + /// The `asahi-g15/pstate` file ([`pstate::write`]): the stats ring's records so far, those + /// it holds unconsumed (read, not consumed), HardwareDataA's words and table A of the + /// device tree's operating points. Read-only: nothing firmware-visible is written. + pub(crate) fn write_pstate(&self, f: &mut impl fmt::Write) -> fmt::Result { + let Ok(hw) = self.pstate_words() else { + return Ok(()); + }; + let seen = self + .channels + .peek_stats() + .unwrap_or(self.channels.stats_seen); + let table = pstate::Table::from_mhz(board::table_a_mhz(&self.session.adt)); + pstate::write(f, &seen, &hw, &table) + } +} diff --git a/drivers/gpu/drm/asahi/g15/power/idle.rs b/drivers/gpu/drm/asahi/g15/power/idle.rs index bb28d71b3945b..0db36fd859e50 100644 --- a/drivers/gpu/drm/asahi/g15/power/idle.rs +++ b/drivers/gpu/drm/asahi/g15/power/idle.rs @@ -14,6 +14,9 @@ //! of works under sustained low-duty submission pay a ~300 us power-up. //! - +0x2c enables the idle decision (template 1, unchanged). //! +//! The standby time comes from [`standby`]: the explicit `g15_idle_standby_us` when set, else the +//! device tree's `apple,idleoff-standby-timer` where the machine has one, else the tuned value. +//! //! Core-only (no `kernel` imports), so that it can be tested on the host. /// Globals offsets (mirrors of `fw::initdata::globals`). @@ -23,6 +26,50 @@ pub(crate) const STANDBY_US: usize = 0x7a8; pub(crate) const MIN_CONF: usize = 0x7c4; pub(crate) const DELAY: usize = 0x9a0; +/// Where the standby time came from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum StandbySource { + /// The `g15_idle_standby_us` module parameter. + Parameter, + /// The GPU node's `apple,idleoff-standby-timer`. + DeviceTree, + /// The tuned value. + Tuning, +} + +impl StandbySource { + pub(crate) const fn name(self) -> &'static str { + match self { + Self::Parameter => "the module parameter", + Self::DeviceTree => "the device tree", + Self::Tuning => "the tuning", + } + } +} + +/// Bounds of an explicit or device-tree standby time, us. +pub(crate) const STANDBY_MIN_US: u32 = 100; +pub(crate) const STANDBY_MAX_US: u32 = 100000; + +/// Chooses the standby time, us: `param` (0: not set, else clamped to the bounds) wins, then +/// `dt` (the device-tree value, if the machine has one and it is within the bounds), else +/// `tuned`. Returns the value and its source; the flag tells a device-tree value outside the +/// bounds was ignored. +pub(crate) fn standby(param: u32, dt: Option, tuned: u32) -> (u32, StandbySource, bool) { + let rejected = matches!(dt, Some(v) if !(STANDBY_MIN_US..=STANDBY_MAX_US).contains(&v)); + if param != 0 { + return ( + param.clamp(STANDBY_MIN_US, STANDBY_MAX_US), + StandbySource::Parameter, + rejected, + ); + } + match dt { + Some(v) if !rejected => (v, StandbySource::DeviceTree, false), + _ => (tuned, StandbySource::Tuning, rejected), + } +} + /// IEEE-754 single bits of `permille / 1000`, `permille` clamped to 1..=1000 (0 cannot be /// expressed: it reads 0.001). pub(crate) const fn permille_f32(permille: u32) -> u32 { @@ -53,6 +100,22 @@ mod tests { x.to_bits() } + #[test] + fn standby_choice() { + use StandbySource::*; + assert_eq!(standby(0, Some(1234), 2500), (1234, DeviceTree, false)); + assert_eq!(standby(0, None, 2500), (2500, Tuning, false)); + assert_eq!(standby(900, Some(1234), 2500), (900, Parameter, false)); + assert_eq!(standby(900, None, 2500), (900, Parameter, false)); + assert_eq!(standby(5, None, 2500), (100, Parameter, false)); + assert_eq!(standby(10_000_000, None, 2500), (100000, Parameter, false)); + assert_eq!(standby(0, Some(0), 2500), (2500, Tuning, true)); + assert_eq!(standby(0, Some(100001), 2500), (2500, Tuning, true)); + assert_eq!(standby(0, Some(100), 2500), (100, DeviceTree, false)); + assert_eq!(standby(0, Some(100000), 2500), (100000, DeviceTree, false)); + assert_eq!(standby(700, Some(0), 2500), (700, Parameter, true)); + } + #[test] fn permille() { assert_eq!(permille_f32(500), 0x3f000000); diff --git a/drivers/gpu/drm/asahi/g15/power/pstate.rs b/drivers/gpu/drm/asahi/g15/power/pstate.rs new file mode 100644 index 0000000000000..411fe22413893 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/power/pstate.rs @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The GPU's current performance state, as the firmware reports it. +//! +//! Two places carry it: +//! +//! - The stats ring: 256 records of 0x40 bytes, a u32 tag at +0 and, in a power-state record +//! (tag 0x0b), the performance-state index at +0x20. +//! - HardwareDataA (hardware record +0x4580): actual (+0x2c), target (+0x30) and current +//! (+0x38) state indices and the frequency in MHz as an IEEE single (+0x58). +//! +//! An index counts the firmware's DVFS states, 0 (off) to 8; its frequency is the state's +//! table-A operating point ([`super::super::board`]), taken from the device tree. +//! +//! [`write`] prints the `asahi-g15/pstate` file. Its first line is what a sampler reads: +//! `pstate freq_khz source `, from the newest power-state record +//! seen, or HardwareDataA's current state when the stats ring has carried none. +//! +//! Core-only (no `kernel` imports), so that it can be tested on the host. + +use core::fmt; + +/// DVFS states of the firmware tables (= `board::DVFS_STATES`). +pub(crate) const STATES: usize = 9; + +/// HardwareDataA words. +pub(crate) const HWDATA_A_ACTUAL: usize = 0x2c; +pub(crate) const HWDATA_A_TARGET: usize = 0x30; +pub(crate) const HWDATA_A_CURRENT: usize = 0x38; +pub(crate) const HWDATA_A_FREQ_MHZ: usize = 0x58; + +/// Stats ring records. +pub(crate) const STATS_RECORD: usize = 0x40; +pub(crate) const STATS_SLOTS: u32 = 256; +pub(crate) const STATS_TAG: usize = 0x00; +pub(crate) const TAG_POWER_STATE: u32 = 0x0b; +pub(crate) const POWER_STATE_INDEX: usize = 0x20; + +/// Tags counted one by one; higher tags are counted together. +pub(crate) const TAGS: usize = 16; + +/// What the stats ring has carried. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct StatsSeen { + /// Records consumed. + pub(crate) records: u64, + /// Records per tag; the last slot counts every tag from `TAGS - 1` up. + pub(crate) tags: [u64; TAGS], + /// The index of the newest power-state record. + pub(crate) last: Option, +} + +impl StatsSeen { + /// Notes one record: its tag and the word at [`POWER_STATE_INDEX`]. + pub(crate) fn note(&mut self, tag: u32, index: u32) { + self.records = self.records.wrapping_add(1); + let slot = (tag as usize).min(TAGS - 1); + self.tags[slot] = self.tags[slot].wrapping_add(1); + if tag == TAG_POWER_STATE { + self.last = Some(index); + } + } +} + +/// The ring slots from the consumer `read` to the producer `write`, oldest first; `None` when +/// either index is out of the ring. +pub(crate) fn pending(read: u32, write: u32) -> Option> { + if read >= STATS_SLOTS || write >= STATS_SLOTS { + return None; + } + let count = (write + STATS_SLOTS - read) % STATS_SLOTS; + Some((0..count).map(move |i| (read + i) % STATS_SLOTS)) +} + +/// The HardwareDataA words. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct HwWords { + pub(crate) actual: u32, + pub(crate) target: u32, + pub(crate) current: u32, + /// IEEE single bits, MHz. + pub(crate) freq_mhz: u32, +} + +/// The table-A frequency of each firmware state, kHz. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Table { + pub(crate) khz: [u32; STATES], +} + +impl Table { + /// From the table-A operating points in MHz. + pub(crate) fn from_mhz(mhz: [u32; STATES]) -> Self { + Self { + khz: mhz.map(|m| m.saturating_mul(1000)), + } + } + + /// The frequency of state `index`, kHz; `None` past the table. + pub(crate) fn khz(&self, index: u32) -> Option { + self.khz.get(index as usize).copied() + } +} + +/// An IEEE single number of MHz in kHz, rounded to nearest; `None` for a negative number, an +/// infinity, a NaN or a value past `u32::MAX` kHz. +pub(crate) fn f32_mhz_to_khz(bits: u32) -> Option { + let exp = (bits >> 23) & 0xff; + let frac = bits & 0x7f_ffff; + if bits & 0x7fff_ffff == 0 { + return Some(0); + } + if bits >> 31 != 0 || exp == 0xff { + return None; + } + // value = m * 2^(e - 150); a subnormal has no implicit bit and e = 1. + let (m, e) = if exp == 0 { + (frac as u64, 1i32) + } else { + ((frac | 0x80_0000) as u64, exp as i32) + }; + let scaled = m * 1000; // < 2^34 + let shift = e - 150; + let khz = if shift >= 0 { + if shift > 31 { + return None; + } + scaled << shift + } else { + let s = (-shift) as u32; + if s >= 64 { + 0 + } else { + let q = scaled >> s; + let rem = scaled & ((1u64 << s) - 1); + let half = 1u64 << (s - 1); + // Ties to even. + if rem > half || (rem == half && q & 1 == 1) { + q + 1 + } else { + q + } + } + }; + u32::try_from(khz).ok() +} + +/// Where the reported state comes from. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Source { + Stats, + HwData, +} + +impl Source { + fn name(self) -> &'static str { + match self { + Source::Stats => "stats", + Source::HwData => "hwdata", + } + } +} + +/// The reported state: the newest power-state record, else HardwareDataA's current state. +pub(crate) fn current(seen: &StatsSeen, hw: &HwWords) -> (u32, Source) { + match seen.last { + Some(index) => (index, Source::Stats), + None => (hw.current, Source::HwData), + } +} + +/// A frequency for the file: kHz, or `unknown`. +struct Khz(Option); + +impl fmt::Display for Khz { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + Some(khz) => write!(f, "{}", khz), + None => f.write_str("unknown"), + } + } +} + +/// The `asahi-g15/pstate` file: +/// +/// ```text +/// pstate freq_khz source +/// stats: records power_states last tags : ... +/// hwdata: actual target current freq_mhz_f32 freq_khz +/// table_khz: x 9 +/// ``` +pub(crate) fn write( + f: &mut impl fmt::Write, + seen: &StatsSeen, + hw: &HwWords, + table: &Table, +) -> fmt::Result { + let (index, source) = current(seen, hw); + writeln!( + f, + "pstate {} freq_khz {} source {}", + index, + Khz(table.khz(index)), + source.name() + )?; + write!( + f, + "stats: records {} power_states {} last ", + seen.records, seen.tags[TAG_POWER_STATE as usize] + )?; + match seen.last { + Some(last) => write!(f, "{}", last)?, + None => f.write_str("none")?, + } + f.write_str(" tags")?; + for (tag, n) in seen.tags.iter().enumerate() { + if *n != 0 { + write!(f, " {}:{}", tag, n)?; + } + } + writeln!(f)?; + writeln!( + f, + "hwdata: actual {} target {} current {} freq_mhz_f32 {:#010x} freq_khz {}", + hw.actual, + hw.target, + hw.current, + hw.freq_mhz, + Khz(f32_mhz_to_khz(hw.freq_mhz)) + )?; + f.write_str("table_khz:")?; + for khz in table.khz { + write!(f, " {}", khz)?; + } + writeln!(f) +} + +#[cfg(test)] +mod tests { + use super::*; + + // A made-up table, not any machine's. + fn table() -> Table { + Table::from_mhz([0, 100, 200, 300, 400, 500, 600, 700, 800]) + } + + #[test] + fn table_is_khz_and_bounded() { + let t = table(); + assert_eq!(t.khz(0), Some(0)); + assert_eq!(t.khz(8), Some(800_000)); + assert_eq!(t.khz(9), None); + assert_eq!(t.khz(u32::MAX), None); + } + + #[test] + fn notes_tags_and_the_newest_power_state() { + let mut s = StatsSeen::default(); + assert_eq!(s.last, None); + s.note(4, 7); + assert_eq!(s.last, None); + s.note(TAG_POWER_STATE, 3); + s.note(TAG_POWER_STATE, 8); + s.note(9, 1); + s.note(200, 0); + assert_eq!(s.records, 5); + assert_eq!(s.last, Some(8)); + assert_eq!(s.tags[4], 1); + assert_eq!(s.tags[TAG_POWER_STATE as usize], 2); + assert_eq!(s.tags[9], 1); + assert_eq!(s.tags[TAGS - 1], 1); + } + + #[test] + fn pending_walks_from_read_to_write_with_wrap() { + let v: Vec = pending(3, 3).unwrap().collect(); + assert!(v.is_empty()); + let v: Vec = pending(3, 6).unwrap().collect(); + assert_eq!(v, [3, 4, 5]); + let v: Vec = pending(254, 1).unwrap().collect(); + assert_eq!(v, [254, 255, 0]); + assert!(pending(256, 0).is_none()); + assert!(pending(0, 256).is_none()); + } + + #[test] + fn f32_mhz_in_khz() { + let k = |x: f32| f32_mhz_to_khz(x.to_bits()); + assert_eq!(k(0.0), Some(0)); + assert_eq!(f32_mhz_to_khz(0x8000_0000), Some(0)); // -0 + assert_eq!(k(1.0), Some(1000)); + assert_eq!(k(800.0), Some(800_000)); + assert_eq!(k(1234.5), Some(1_234_500)); + assert_eq!(k(0.0015), Some(2)); // 1.5 kHz, nearest of the single + assert_eq!(k(0.0004), Some(0)); + assert_eq!(k(4_000_000.0), Some(4_000_000_000)); + assert_eq!(k(5_000_000.0), None); // past u32 kHz + assert_eq!(k(-1.0), None); + assert_eq!(k(f32::INFINITY), None); + assert_eq!(k(f32::NAN), None); + assert_eq!(f32_mhz_to_khz(1), Some(0)); // smallest subnormal + // Exact halves (an odd mantissa x 2^-4 MHz is n + 0.5 kHz) round to even. + for m in [0x80_0001u32, 0x80_0003, 0x80_0005, 0xff_ffff] { + let bits = (146 << 23) | (m & 0x7f_ffff); + let want = (m as f64 * 62.5).round_ties_even() as u32; + assert_eq!(f32_mhz_to_khz(bits), Some(want), "{m:#x}"); + } + // Against the host's arithmetic over a sweep. + for i in 0..20_000u32 { + let x = i as f32 * 0.37; + let want = (x as f64 * 1000.0).round_ties_even() as u32; + assert_eq!(k(x), Some(want), "{x}"); + } + } + + #[test] + fn current_prefers_the_stats_ring() { + let hw = HwWords { + current: 5, + ..Default::default() + }; + let mut s = StatsSeen::default(); + assert_eq!(current(&s, &hw), (5, Source::HwData)); + s.note(TAG_POWER_STATE, 8); + assert_eq!(current(&s, &hw), (8, Source::Stats)); + s.note(TAG_POWER_STATE, 0); + assert_eq!(current(&s, &hw), (0, Source::Stats)); + } + + #[test] + fn file_from_the_stats_ring() { + let mut s = StatsSeen::default(); + s.note(4, 0); + s.note(TAG_POWER_STATE, 8); + let hw = HwWords { + actual: 8, + target: 8, + current: 6, + freq_mhz: 800.0f32.to_bits(), + }; + let mut out = String::new(); + write(&mut out, &s, &hw, &table()).unwrap(); + assert_eq!( + out, + "pstate 8 freq_khz 800000 source stats\n\ + stats: records 2 power_states 1 last 8 tags 4:1 11:1\n\ + hwdata: actual 8 target 8 current 6 freq_mhz_f32 0x44480000 freq_khz 800000\n\ + table_khz: 0 100000 200000 300000 400000 500000 600000 700000 800000\n" + ); + } + + #[test] + fn file_from_hwdata_and_an_index_past_the_table() { + let hw = HwWords { + actual: 1, + target: 2, + current: 12, + freq_mhz: f32::NAN.to_bits(), + }; + let mut out = String::new(); + write(&mut out, &StatsSeen::default(), &hw, &table()).unwrap(); + let mut lines = out.lines(); + assert_eq!( + lines.next(), + Some("pstate 12 freq_khz unknown source hwdata") + ); + assert_eq!( + lines.next(), + Some("stats: records 0 power_states 0 last none tags") + ); + assert_eq!( + lines.next(), + Some("hwdata: actual 1 target 2 current 12 freq_mhz_f32 0x7fc00000 freq_khz unknown") + ); + } +} diff --git a/drivers/gpu/drm/asahi/g15/recovery.rs b/drivers/gpu/drm/asahi/g15/recovery.rs index b977403384d50..0827385e92c5f 100644 --- a/drivers/gpu/drm/asahi/g15/recovery.rs +++ b/drivers/gpu/drm/asahi/g15/recovery.rs @@ -57,7 +57,7 @@ use kernel::{ use logic::{Failure, HangAction, HangInputs}; use super::fw::channels::{devctl, DevCtl, RING_SLOTS}; -use super::fw::initdata::{control, globals, hwdata}; +use super::fw::initdata::{control, globals}; use super::hw::{sizes, tuning}; use super::job::{Completions, Job}; use super::{params, Firmware}; @@ -530,7 +530,7 @@ impl Firmware { self.init .hwdata .as_ref() - .and_then(|hw| hw.r32(hwdata::FAULT_INFO + offset).ok()) + .and_then(|hw| hw.r32(self.cfg.layout.hwdata_view + offset).ok()) .unwrap_or(0) }; dev_info!( diff --git a/drivers/gpu/drm/asahi/g15/regs.rs b/drivers/gpu/drm/asahi/g15/regs.rs index 886057b07225c..474e4aa5c3a70 100644 --- a/drivers/gpu/drm/asahi/g15/regs.rs +++ b/drivers/gpu/drm/asahi/g15/regs.rs @@ -51,6 +51,8 @@ enum Kind { Device, /// `memremap(MEMREMAP_WC)`: normal non-cacheable memory. WriteCombined, + /// `memremap(MEMREMAP_WB)`: System RAM through the kernel's own mapping. + WriteBack, } /// A CPU mapping of a physical range that is not System RAM, accessed with the `readl`/`writel` @@ -115,6 +117,30 @@ impl Mapping { }) } + /// Maps a boot-loader region the firmware shares with the host: write-combined when it is + /// a carveout outside System RAM (the T6030 regions), through the kernel's cacheable + /// mapping when it is a reserved-memory region inside System RAM (the T6031 regions, which + /// the earlier driver of that SoC used the same way). + pub(crate) fn region(phys: u64, size: usize) -> Result { + if size == 0 { + return Err(EINVAL); + } + if !super::boot::glue::ram_within(phys, size as u64) { + return Self::write_combined(phys, size); + } + // SAFETY: memremap() of System RAM with MEMREMAP_WB returns its existing mapping; the + // range is a reserved-memory region of the GPU node that nothing else uses. + let addr = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WB as _) }; + if addr.is_null() { + return Err(ENOMEM); + } + Ok(Self { + raw: MmioRaw::new(addr as usize, size)?, + phys, + kind: Kind::WriteBack, + }) + } + /// The `readl`/`writel` accessors of this mapping. pub(crate) fn io(&self) -> &Mmio { // SAFETY: By the type invariant `raw` describes a live mapping of `maxsize` bytes. @@ -133,20 +159,27 @@ impl Drop for Mapping { unsafe { match self.kind { Kind::Device => bindings::iounmap(self.raw.addr() as *mut _), - Kind::WriteCombined => bindings::memunmap(self.raw.addr() as *mut _), + Kind::WriteCombined | Kind::WriteBack => { + bindings::memunmap(self.raw.addr() as *mut _) + } } } } } +/// Most clusters of a supported SoC. +pub(crate) const MAX_CLUSTERS: usize = 4; + /// Identification read at start. #[derive(Copy, Clone, Default)] pub(crate) struct Identity { pub(crate) version: u32, pub(crate) counts: u32, - /// Raw enabled-core mask and its per-cluster split (10 bits each). - pub(crate) cores: u32, - pub(crate) core_masks: [u32; 2], + /// Raw enabled-core mask (`+0xe01500`, and `+0xe01504` above 32 core slots) and its + /// per-cluster split ([`super::hw::CLUSTER_BITS`] each, cluster 0 lowest), one entry per + /// cluster of the SoC. + pub(crate) cores: u64, + pub(crate) core_masks: [u32; MAX_CLUSTERS], } /// Engine receipts: busy bit and the two fault banks. @@ -197,17 +230,30 @@ impl Regs { }) } - /// Reads the identification words and the enabled-core mask (after the AGX vote). - pub(crate) fn identity(&self) -> Result { + /// Reads the identification words and the enabled-core mask (once the GPU is powered). A + /// SoC with more than 32 core slots carries the mask in two words. + pub(crate) fn identity(&self, cfg: &super::Config) -> Result { let version = self.res.sgx_read32_at(SGX_ID)?; let counts = self.res.sgx_read32_at(SGX_ID + 0x10)?; - let cores = self.res.sgx_read32_at(SGX_CORE + 0x1500)?; - let mask = (1u32 << super::hw::CLUSTER_BITS) - 1; + let mut cores = u64::from(self.res.sgx_read32_at(SGX_CORE + 0x1500)?); + if cfg.core_slots() > 32 { + cores |= u64::from(self.res.sgx_read32_at(SGX_CORE + 0x1504)?) << 32; + } + let bits = super::hw::CLUSTER_BITS; + let mask = (1u64 << bits) - 1; + let mut core_masks = [0u32; MAX_CLUSTERS]; + for (cluster, slot) in core_masks + .iter_mut() + .enumerate() + .take((cfg.num_clusters as usize).min(MAX_CLUSTERS)) + { + *slot = ((cores >> (cluster as u32 * bits)) & mask) as u32; + } Ok(Identity { version, counts, cores, - core_masks: [cores & mask, (cores >> super::hw::CLUSTER_BITS) & mask], + core_masks, }) } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index bc5438a5ac0fb..152ed5cdd02d8 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -5,7 +5,8 @@ //! The firmware's kernel root (the TABLES region, ADT `gpu-region`) and the shared L2 page are //! not System RAM, so the shared page-table code cannot walk them. The design: //! -//! 1. Map handoff, TTBAT, kernel root page and shared L2 write-combined. PPL exchange: write the +//! 1. Map handoff, TTBAT, kernel root page and shared L2 (write-combined carveouts, or the +//! kernel's mapping of reserved System RAM, [`Mapping::region`]). PPL exchange: write the //! AP magic, `cur_slot = 0xffffffff`, zero the 65 flush entries and `+0x640`, then poll the //! firmware magic (1 s). //! 2. `mmu::Uat::new_adopted` with `kernel_root = TABLES`: user VMs and the kernel lower VM get @@ -197,13 +198,13 @@ pub(crate) struct Window { } impl Window { - /// Maps the UAT handoff regions write-combined. + /// Maps the UAT handoff regions ([`Mapping::region`]). pub(crate) fn new(adt: &super::adt::GpuAdt) -> Result { Ok(Self { - handoff: Mapping::write_combined(adt.handoff.start, hw::HANDOFF_SIZE as usize)?, - ttbs: Mapping::write_combined(adt.ttbs.start, hw::TTBS_SIZE as usize)?, - root: Mapping::write_combined(adt.tables.start, hw::PAGE as usize)?, - shared_l2: Mapping::write_combined(adt.shared_l2.start, hw::SHARED_L2_SIZE as usize)?, + handoff: Mapping::region(adt.handoff.start, hw::HANDOFF_SIZE as usize)?, + ttbs: Mapping::region(adt.ttbs.start, hw::TTBS_SIZE as usize)?, + root: Mapping::region(adt.tables.start, hw::PAGE as usize)?, + shared_l2: Mapping::region(adt.shared_l2.start, hw::SHARED_L2_SIZE as usize)?, image: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?, snapshot: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?, }) diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index 1b3235ce2365c..bd7496203001d 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -10,6 +10,8 @@ //! ([`object::Objects::clients`]) as driver mappings at their fixed GVAs, and reserve //! [`super::hw::client::CLIENT_APERTURE`], the pass-slot range and the PB page list //! (`vm.install_driver_mappings`): user binds there fail (EINVAL); +//! - map the PB heap's backed chunks at their fixed GVAs ([`VmState::map_heap`]; chunks backed +//! later are mapped at the VM's publications and, for a VM with a flight, at the refill); //! - bind it to an exclusive TTBAT slot ([`mmu::Uat::bind`]) for as long as its [`VmState`] //! lives; at most [`super::hw::sizes::MAX_VMS`] such VMs (ENOSPC beyond, checked before //! binding so the slot allocator never blocks). @@ -102,6 +104,20 @@ impl ClientAlias { } } +/// A PB heap chunk mapped in a VM at its fixed GVA ([`super::pools::chunk_gva`]). +struct HeapMapping { + gem: ARef, + _mapping: mmu::KernelMapping, +} + +/// The PB heap chunks mapped in a VM, in heap order (chunk `i` at `chunk_gva(i)`). Taken +/// under the device lock (refill, publication, reclaim) or by the creating thread. +#[pin_data] +struct HeapMappings { + #[pin] + list: Mutex>, +} + /// G15 state of one user VM, shared by the registry, its queues and its jobs. pub(crate) struct VmState { /// The `mmu::Vm` identifier (`Gpu::ids.vm`). @@ -112,6 +128,8 @@ pub(crate) struct VmState { pub(crate) status: Arc, /// The VM: span checks, stream reads and job guards. pub(crate) vm: mmu::Vm, + /// The PB heap chunks mapped in this VM. + heap: Arc, /// The exclusive slot binding, released before the live count drops. bind: Option, registry: Arc, @@ -125,6 +143,71 @@ impl Drop for VmState { } } +impl VmState { + /// Maps PB heap chunk `i` (a chunk this VM has not seen: the next one) at its GVA. + fn map_one( + vm: &mmu::Vm, + heap: &mut KVec, + i: usize, + gem: ARef, + ) -> Result { + if i != heap.len() { + return Err(EINVAL); + } + let mapping = vm.map_at( + super::pools::chunk_gva(i), + super::pools::CHUNK_BYTES as usize, + gem.clone(), + mmu::Prot::from_pte(hw::pte::LEAF_CACHED), + false, + )?; + heap.push( + HeapMapping { + gem, + _mapping: mapping, + }, + GFP_KERNEL, + )?; + Ok(()) + } + + /// Maps the backed chunks (`gems`, in heap order) this VM does not have yet. A chunk whose + /// object changed since (freed and backed again) is unmapped first, with everything above + /// it. + pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result { + let mut heap = self.heap.list.lock(); + let mut n = 0; + for (i, gem) in gems.enumerate() { + n = i + 1; + if let Some(have) = heap.get(i) { + if core::ptr::eq(&*have.gem, &*gem) { + continue; + } + heap.truncate(i); + } + Self::map_one(&self.vm, &mut heap, i, gem)?; + } + // Chunks freed since this VM last mapped the heap. + heap.truncate(n); + Ok(()) + } + + /// Unmaps the heap chunks from index `keep` on (freed by a reclaim). + pub(crate) fn unmap_heap_above(&self, keep: usize) { + self.heap.list.lock().truncate(keep); + } + + /// Maps the newly backed chunk `i` into this VM (a VM with a flight at a refill). + pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result { + let mut heap = self.heap.list.lock(); + if i < heap.len() { + // Already mapped (publication caught up first). + return Ok(()); + } + Self::map_one(&self.vm, &mut heap, i, gem) + } +} + /// Live-VM accounting and the VMs whose client handle is open. #[pin_data] pub(crate) struct Registry { @@ -237,6 +320,17 @@ impl Registry { .ok_or(ENOENT) } + /// The VMs whose client handle is open (and the closed ones not pruned yet), for a walk + /// that takes VM locks: not under the registry lock. + pub(crate) fn open(&self) -> Result>> { + let vms = self.vms.lock(); + let mut out = KVec::with_capacity(vms.len(), GFP_KERNEL)?; + for state in vms.iter() { + out.push(state.clone(), GFP_KERNEL)?; + } + Ok(out) + } + /// Latches `error` on every VM whose client handle is open. The fail-stop path calls it under /// the device lock. pub(crate) fn fail_all(&self, error: Error) { @@ -263,13 +357,16 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { if gpu.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } - // Snapshot the registry and the client aliases; map them without the device lock. - let (registry, aliases) = { + // Snapshot the registry, the client aliases and the backed PB heap chunks; map them without + // the device lock (a chunk backed after the snapshot is mapped at the VM's first + // publication). + let (registry, aliases, chunks) = { let state = gpu.shared.state.lock(); let firmware = state.as_ref().ok_or(ENODEV)?; ( firmware.vms.clone(), ClientAlias::collect(&firmware.objects)?, + firmware.pools.heap_gems()?, ) }; let reservation = registry.reserve()?; @@ -278,6 +375,16 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { let status = vm.status().ok_or(EINVAL)?.clone(); let (mappings, reserved) = Registry::driver_mappings(&vm, aliases)?; vm.install_driver_mappings(mappings, reserved)?; + let mut list = KVec::with_capacity(chunks.len(), GFP_KERNEL)?; + for (i, gem) in chunks.into_iter().enumerate() { + VmState::map_one(&vm, &mut list, i, gem)?; + } + let heap = Arc::pin_init( + pin_init!(HeapMappings { + list <- new_mutex!(list, "g15::HeapMappings::list"), + }), + GFP_KERNEL, + )?; // The context names the root only now: everything above is already visible to the walker. let bind = gpu.uat.bind(&vm)?; let slot = bind.slot(); @@ -289,6 +396,7 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { slot, status, vm: vm.clone(), + heap, bind: Some(bind), registry: registry.clone(), }, diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 15d1585579c68..f4e1994a75fbb 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2298,9 +2298,22 @@ impl Uat { // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; + // A carveout outside System RAM is mapped write-combined; a reserved-memory region + // inside System RAM (mapped by the kernel) through the kernel's own mapping, as + // `map_region` does for the other generations. + extern "C" { + fn asahi_g15_ram_within(start: u64, size: u64) -> bool; + } + // SAFETY: Only queries the resource tree (`g15/glue.c`). + let ram = unsafe { asahi_g15_ram_within(range.start, size as u64) }; + let flags = if ram { + io::mem::MemFlag::WB + } else { + io::mem::MemFlag::WC + }; // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU // reaches them through its own translation. - let map = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WC.into()) }?; + let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, map, diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 23ec383c487f0..d5cc2c354f0d6 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -34,6 +34,8 @@ pmp-y := pmp-core.o pmp-display.o # Must run before devices are populated, so built in even when pmp is a module. # It adds device tree nodes, so it needs OF_DYNAMIC. obj-$(if $(CONFIG_APPLE_PMP),$(CONFIG_OF_DYNAMIC)) += j514s-pmp-handoff.o +# The T6031 handoff enables the display nodes and adds the PMP the same way. +obj-$(CONFIG_DRM_APPLE_DCP_T6031) += t6031-display-handoff.o obj-$(CONFIG_APPLE_MCC) += apple-mcc.o apple-mcc-y := mcc.o diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 3d76ebc59d51f..a7268dea8a0fb 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -883,6 +883,7 @@ static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, + { .compatible = "apple,t6031-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, { .compatible = "apple,asc-mailbox-v4", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8015-asc-mailbox", .data = &apple_mbox_t8015_hw }, { .compatible = "apple,m3-mailbox-v2", .data = &apple_mbox_m3_hw }, diff --git a/drivers/soc/apple/pmp-display.c b/drivers/soc/apple/pmp-display.c index e4d8468d8365c..18aab24e87864 100644 --- a/drivers/soc/apple/pmp-display.c +++ b/drivers/soc/apple/pmp-display.c @@ -1,14 +1,17 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * T6030 (M3 Pro: J514S, J516S) power votes through pmp-core.rs's RTKit session. + * T6030 (M3 Pro: J514S, J516S) and T6031 (M3 Max: J514C, J516C) power votes + * through pmp-core.rs's RTKit session. * * The display is inherited running from the boot loader. Before the PMP CPU * runs, its power chain is held and the PMP's DISP|ANS request is seeded in * the PTD, so the PMP keeps it on; the DCP driver then waits for the PMP to - * acknowledge it. The GPU's AGX vote is an untagged RTKit request on the PMP - * endpoint: exactly one is in flight, with a one-second deadline. Votes and - * supplier links are retained until reboot. Malformed, unsolicited, late or - * duplicate replies and firmware crashes latch a sticky error. + * acknowledge it. Where the table and the power chain sit differs per SoC + * (struct pmp_display_soc). The GPU's AGX vote, T6030 only, is an untagged + * RTKit request on the PMP endpoint: exactly one is in flight, with a + * one-second deadline. Votes and supplier links are retained until reboot. + * Malformed, unsolicited, late or duplicate replies and firmware crashes + * latch a sticky error. */ #include #include @@ -97,31 +100,153 @@ struct apple_pmp_display { static struct apple_pmp_display *pmp_display; static DEFINE_MUTEX(pmp_display_registration); +/* + * J514S (T6030). PTD at 0x3503c0000: status +0x10; DISP request word +280*16 + * and ACK +284*16 (PTD ranges 10/11); non-posted request-set aperture + * +0x10000 + 280*8. DISP is table index 7, ANS index 16. + */ +#define J514S_PTD 0x3503c0000ULL +#define J514S_PTD_SIZE 0x14000 +#define J514S_PTD_STATUS 0x10 +#define J514S_PTD_REQ (280 * 16) +#define J514S_PTD_ACK (284 * 16) +#define J514S_PTD_SET (0x10000 + 280 * 8) +#define J514S_PTD_DISP BIT_ULL(7) +#define J514S_PTD_ANS BIT_ULL(16) +#define J514S_PMP_ASC 0x350c00000ULL +#define J514S_PMGR 0x350700000ULL + +/* SYS, FE and CPU of the display power chain, parents first. */ +static const u32 j514s_display_ps[] = { 0x1c0, 0x258, 0x10000 }; + +/* + * J516C/J514C (T6031). PTD at 0x2903c0000 (the PMP's ptd-update window less + * 0x10000): status +0x10; the SoC device request word at PTD index 0x200 + * (+0x2000), its acknowledgement at index 0x208 (+0x2080), the request-set + * aperture at +0x10000 + 0x200*8. The internal display is SoC device 17, + * request bit 16; storage (ANS) is device 34, bit 33. The display chain + * sits in one PMGR block: DISP_SYS +0x200, DISP_FE +0x298, DISP_CPU +0x2a0. + */ +#define J516C_PTD 0x2903c0000ULL +#define J516C_PTD_SIZE 0x14000 +#define J516C_PTD_REQ (0x200 * 16) +#define J516C_PTD_ACK (0x208 * 16) +#define J516C_PTD_SET (0x10000 + 0x200 * 8) +#define J516C_PTD_DISP BIT_ULL(16) +#define J516C_PTD_ANS BIT_ULL(33) +#define J516C_PMP_ASC 0x290c00000ULL +#define J516C_PMGR 0x292280000ULL + +static const u32 j516c_display_ps[] = { 0x200, 0x298, 0x2a0 }; + +/* + * One SoC's PMP as the display hold sees it: the node the boot-time handoff + * adds, its device-state table and the display power chain it must keep up. + */ +struct pmp_display_soc { + const char *name; + const char *pmp_path; + const char *pmp_compat; + u64 ptd; + u32 ptd_size; + u32 ptd_status; + u32 ptd_req, ptd_ack, ptd_set; + u64 ptd_disp, ptd_ans; + u64 asc; + u64 pmgr; + u32 pmgr_size; + const u32 *display_ps; + unsigned int nr_display_ps; + /* The CPU-client memory floor and the GPU vote are known on T6030 only. */ + bool memory_floor; +}; + +static const struct pmp_display_soc pmp_display_t6030 = { + .name = "J514S", + .pmp_path = "/soc/pmp@350500000", + .pmp_compat = "apple,t6030-pmp-v2", + .ptd = J514S_PTD, + .ptd_size = J514S_PTD_SIZE, + .ptd_status = J514S_PTD_STATUS, + .ptd_req = J514S_PTD_REQ, + .ptd_ack = J514S_PTD_ACK, + .ptd_set = J514S_PTD_SET, + .ptd_disp = J514S_PTD_DISP, + .ptd_ans = J514S_PTD_ANS, + .asc = J514S_PMP_ASC, + .pmgr = J514S_PMGR, + .pmgr_size = 0x14000, + .display_ps = j514s_display_ps, + .nr_display_ps = ARRAY_SIZE(j514s_display_ps), + .memory_floor = true, +}; + +static const struct pmp_display_soc pmp_display_t6031 = { + .name = "J516C", + .pmp_path = "/soc/pmp@290700000", + .pmp_compat = "apple,t6031-pmp-v2", + .ptd = J516C_PTD, + .ptd_size = J516C_PTD_SIZE, + .ptd_status = J514S_PTD_STATUS, + .ptd_req = J516C_PTD_REQ, + .ptd_ack = J516C_PTD_ACK, + .ptd_set = J516C_PTD_SET, + .ptd_disp = J516C_PTD_DISP, + .ptd_ans = J516C_PTD_ANS, + .asc = J516C_PMP_ASC, + .pmgr = J516C_PMGR, + .pmgr_size = 0xc000, + .display_ps = j516c_display_ps, + .nr_display_ps = ARRAY_SIZE(j516c_display_ps), +}; + +/* The running board's PMP description, or NULL where no display is held. */ +static const struct pmp_display_soc *pmp_display_soc(void) +{ + if (apple_pmp_t6030_board()) + return &pmp_display_t6030; + if (apple_pmp_t6031_board()) + return &pmp_display_t6031; + return NULL; +} + bool apple_pmp_display_retained_board(void) { - return apple_pmp_t6030_board(); + return !!pmp_display_soc(); } -/* The board's PMP node, added by the T6030 PMP handoff (no alias). */ +/* The board's PMP node, added by the boot-time PMP handoff (no alias). */ static struct device_node *pmp_bridge_node(void) { - return of_find_node_by_path("/soc/pmp@350500000"); + const struct pmp_display_soc *soc = pmp_display_soc(); + + return soc ? of_find_node_by_path(soc->pmp_path) : NULL; } +bool apple_pmp_display_present(void) +{ + struct device_node *np = pmp_bridge_node(); + + of_node_put(np); + return !!np; +} +EXPORT_SYMBOL_GPL(apple_pmp_display_present); + struct apple_pmp_display *apple_pmp_display_register(struct device *dev, const void *cookie, int (*send)(const void *, u64)) { + const struct pmp_display_soc *soc = pmp_display_soc(); struct apple_pmp_display *p = NULL; struct device_node *node; bool match; - if (!apple_pmp_t6030_board() || !dev->of_node || - !of_device_is_compatible(dev->of_node, "apple,t6030-pmp-v2")) + if (!soc || !dev->of_node || + !of_device_is_compatible(dev->of_node, soc->pmp_compat)) return NULL; node = pmp_bridge_node(); match = node == dev->of_node; of_node_put(node); - /* The handoff copies the soc-device table from the ADT; require it. */ + /* The handoff copies the soc-device table from the boot loader; require it. */ if (!match || !of_property_present(dev->of_node, "apple,tunable-soc-device") || !cookie || !send) return NULL; @@ -142,7 +267,7 @@ struct apple_pmp_display *apple_pmp_display_register(struct device *dev, __module_get(THIS_MODULE); pm_runtime_get_noresume(dev); smp_store_release(&pmp_display, p); - dev_info(dev, "J514S PMP vote bridge ready; retained existing PMP owner\n"); + dev_info(dev, "%s PMP vote bridge ready; retained existing PMP owner\n", soc->name); out: mutex_unlock(&pmp_display_registration); return p; @@ -327,62 +452,47 @@ int apple_pmp_j514s_gpu_check(struct device *consumer) EXPORT_SYMBOL_GPL(apple_pmp_j514s_gpu_check); /* - * J514S (T6030). PTD at 0x3503c0000: status +0x10; DISP request word +280*16 - * and ACK +284*16 (PTD ranges 10/11); non-posted request-set aperture - * +0x10000 + 280*8. DISP is table index 7, ANS index 16. - */ -#define J514S_PTD 0x3503c0000ULL -#define J514S_PTD_SIZE 0x14000 -#define J514S_PTD_STATUS 0x10 -#define J514S_PTD_REQ (280 * 16) -#define J514S_PTD_ACK (284 * 16) -#define J514S_PTD_SET (0x10000 + 280 * 8) -#define J514S_PTD_DISP BIT_ULL(7) -#define J514S_PTD_ANS BIT_ULL(16) -/* - * Memory floor: CPU-client EACC0/PACC0 PTD indices 288/289 request DCS4 and - * FAB2 (valid bit 61, DCS bits 51:48, FAB bits 35:32), other fields kept. - * DVFS state entries 8 and 10 (PTD range 3) publish the firmware's response. + * Memory floor (T6030): CPU-client EACC0/PACC0 PTD indices 288/289 request + * DCS4 and FAB2 (valid bit 61, DCS bits 51:48, FAB bits 35:32), other fields + * kept. DVFS state entries 8 and 10 (PTD range 3) publish the firmware's + * response. The T6031 table's CPU-client entries have not been read, so no + * floor is requested there. */ #define J514S_PTD_FLOOR_IDX0 288 #define J514S_PTD_FLOOR_MASK (BIT_ULL(61) | GENMASK_ULL(51, 48) | GENMASK_ULL(35, 32)) #define J514S_PTD_FLOOR (BIT_ULL(61) | (4ULL << 48) | (2ULL << 32)) #define J514S_PTD_FLOOR_SET(i) (0x10000 + (J514S_PTD_FLOOR_IDX0 + (i)) * 8) #define J514S_PTD_DVFS_STATE(i) ((8 + 2 * (i)) * 16) -#define J514S_PMP_ASC 0x350c00000ULL -#define J514S_PMGR 0x350700000ULL - -/* SYS, FE and CPU of the display power chain, parents first. */ -static const u32 j514s_display_ps[] = { 0x1c0, 0x258, 0x10000 }; /* * Hold the inherited display epoch: minimum state 15 (bits 19:16), W1C history * bits 9:8 written as zero, other controls preserved. Must precede PMP RUN, * or the PMP powers the display down and the DCP CPU stops. */ -static int j514s_display_hold(struct device *dev) +static int pmp_display_hold(struct device *dev, const struct pmp_display_soc *soc) { - void __iomem *pmgr = ioremap_np(J514S_PMGR, 0x14000); + void __iomem *pmgr = ioremap_np(soc->pmgr, soc->pmgr_size); int ret = 0; u32 val; if (!pmgr) return -ENOMEM; - for (unsigned int i = 0; i < ARRAY_SIZE(j514s_display_ps); i++) { - val = readl(pmgr + j514s_display_ps[i]); + for (unsigned int i = 0; i < soc->nr_display_ps; i++) { + val = readl(pmgr + soc->display_ps[i]); if ((val & 0xff) != 0xff || (val & BIT(31))) { - dev_err(dev, "J514S display domain %#x not inherited on: %#x\n", - j514s_display_ps[i], val); + dev_err(dev, "%s display domain %#x not inherited on: %#x\n", + soc->name, soc->display_ps[i], val); ret = -EIO; break; } - writel((val & ~0x000f0300) | 0x000f0000, pmgr + j514s_display_ps[i]); - val = readl(pmgr + j514s_display_ps[i]); - dev_info(dev, "J514S display domain %#x held: %#x\n", j514s_display_ps[i], val); + writel((val & ~0x000f0300) | 0x000f0000, pmgr + soc->display_ps[i]); + val = readl(pmgr + soc->display_ps[i]); + dev_info(dev, "%s display domain %#x held: %#x\n", soc->name, + soc->display_ps[i], val); /* Not every domain implements the minimum field (SYS reads 0). */ if ((val & 0x000f0000) != 0x000f0000) - dev_warn(dev, "J514S display domain %#x minimum not latched\n", - j514s_display_ps[i]); + dev_warn(dev, "%s display domain %#x minimum not latched\n", + soc->name, soc->display_ps[i]); } iounmap(pmgr); return ret; @@ -390,43 +500,45 @@ static int j514s_display_hold(struct device *dev) int apple_pmp_display_pre_run(struct device *dev) { + const struct pmp_display_soc *soc = pmp_display_soc(); void __iomem *asc = NULL, *ptd = NULL; - u64 status, req; + u64 status, req, seed; u32 run; int ret; - if (!apple_pmp_t6030_board() || !dev->of_node || - !of_device_is_compatible(dev->of_node, "apple,t6030-pmp-v2")) + if (!soc || !dev->of_node || + !of_device_is_compatible(dev->of_node, soc->pmp_compat)) return 0; /* The display floor first: the DCP must survive PMP startup. */ - ret = j514s_display_hold(dev); + ret = pmp_display_hold(dev, soc); if (ret) return ret; - asc = ioremap_np(J514S_PMP_ASC, 0x4000); - ptd = ioremap_np(J514S_PTD, J514S_PTD_SIZE); + asc = ioremap_np(soc->asc, 0x4000); + ptd = ioremap_np(soc->ptd, soc->ptd_size); if (!asc || !ptd) { ret = -ENOMEM; goto out; } run = readl(asc + 0x44); - status = readq(ptd + J514S_PTD_STATUS); - req = readq(ptd + J514S_PTD_REQ); + status = readq(ptd + soc->ptd_status); + req = readq(ptd + soc->ptd_req); /* Cold start only: no adoption of a running or pre-seeded PMP. */ if ((run & BIT(4)) || status || req) { - dev_err(dev, "J514S PMP not cold (run=%#x status=%#llx req=%#llx)\n", - run, status, req); + dev_err(dev, "%s PMP not cold (run=%#x status=%#llx req=%#llx)\n", + soc->name, run, status, req); ret = -EBUSY; goto out; } /* One 64-bit set transaction; the readback must be exactly DISP|ANS. */ - writeq(J514S_PTD_DISP | J514S_PTD_ANS, ptd + J514S_PTD_SET); - req = readq(ptd + J514S_PTD_REQ); - if (req != (J514S_PTD_DISP | J514S_PTD_ANS)) { - dev_err(dev, "J514S PTD seed not accepted: req=%#llx\n", req); + seed = soc->ptd_disp | soc->ptd_ans; + writeq(seed, ptd + soc->ptd_set); + req = readq(ptd + soc->ptd_req); + if (req != seed) { + dev_err(dev, "%s PTD seed not accepted: req=%#llx\n", soc->name, req); ret = -EIO; goto out; } - dev_info(dev, "J514S display held, PTD seeded DISP|ANS: req=%#llx\n", req); + dev_info(dev, "%s display held, PTD seeded DISP|ANS: req=%#llx\n", soc->name, req); out: if (ptd) iounmap(ptd); @@ -505,36 +617,36 @@ static void j514s_memory_floor(void) schedule_delayed_work(&j514s_floor_work, HZ); } -static int j514s_display_acked(bool wait) +static int pmp_display_acked(const struct pmp_display_soc *soc, bool wait) { - void __iomem *ptd = ioremap_np(J514S_PTD, J514S_PTD_SIZE); + void __iomem *ptd = ioremap_np(soc->ptd, soc->ptd_size); u64 status; int ret; if (!ptd) return -ENOMEM; - ret = readq_poll_timeout(ptd + J514S_PTD_STATUS, status, status == 1, + ret = readq_poll_timeout(ptd + soc->ptd_status, status, status == 1, 1000, wait ? 2000000 : 1); /* * DISP must be acknowledged (an ACK implies the request; ANS needs * none). The request entry is not read: that may consume the PMP's * change notification of an external controller's request. */ - if (!ret && !(readq(ptd + J514S_PTD_ACK) & J514S_PTD_DISP)) + if (!ret && !(readq(ptd + soc->ptd_ack) & soc->ptd_disp)) ret = -EIO; iounmap(ptd); - if (!ret && wait) + if (!ret && wait && soc->memory_floor) j514s_memory_floor(); return ret; } -int apple_pmp_j514s_display_prepare(struct device *consumer) +int apple_pmp_display_prepare(struct device *consumer) { struct device_node *np; struct platform_device *pmp; bool bound; - if (!apple_pmp_t6030_board()) + if (!pmp_display_soc()) return -ENODEV; np = pmp_bridge_node(); pmp = np ? of_find_device_by_node(np) : NULL; @@ -547,16 +659,20 @@ int apple_pmp_j514s_display_prepare(struct device *consumer) put_device(&pmp->dev); return bound ? 0 : -EPROBE_DEFER; } -EXPORT_SYMBOL_GPL(apple_pmp_j514s_display_prepare); +EXPORT_SYMBOL_GPL(apple_pmp_display_prepare); -int apple_pmp_j514s_display_get(struct device *consumer) +int apple_pmp_display_get(struct device *consumer) { - return apple_pmp_t6030_board() ? j514s_display_acked(true) : -ENODEV; + const struct pmp_display_soc *soc = pmp_display_soc(); + + return soc ? pmp_display_acked(soc, true) : -ENODEV; } -EXPORT_SYMBOL_GPL(apple_pmp_j514s_display_get); +EXPORT_SYMBOL_GPL(apple_pmp_display_get); -int apple_pmp_j514s_display_check(struct device *consumer) +int apple_pmp_display_check(struct device *consumer) { - return apple_pmp_t6030_board() ? j514s_display_acked(false) : -ENODEV; + const struct pmp_display_soc *soc = pmp_display_soc(); + + return soc ? pmp_display_acked(soc, false) : -ENODEV; } -EXPORT_SYMBOL_GPL(apple_pmp_j514s_display_check); +EXPORT_SYMBOL_GPL(apple_pmp_display_check); diff --git a/drivers/soc/apple/t6031-display-handoff.c b/drivers/soc/apple/t6031-display-handoff.c new file mode 100644 index 0000000000000..5f88da65d8399 --- /dev/null +++ b/drivers/soc/apple/t6031-display-handoff.c @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * T6031 (M3 Max: J514C, J516C) internal display handoff. + * + * The boot loader leaves the internal DCP running with the panel lit. On + * these boards it describes that state in the device tree without enabling + * anything: the DCP, its mailbox, the two display DARTs and the display + * subsystem stay "disabled", and the DCP, the display subsystem and the + * PIODMA child carry apple,t6031-handoff = <1> once the inherited DART + * mappings have been checked and locked. The firmware's memory sits in + * no-map reservations named by memory-region. This Mac's PMP values (board + * and DRAM ids, the PMP tunables) are in /chosen/asahi,t6031-pmp. + * + * Before devices are populated, and only with apple_t6031_display.enable=1 + * on the command line, this file completes that description for the common + * Apple display driver: + * - it adds the PMP, its DART and its mailbox under /soc with this Mac's + * values, as the T6030 handoff does from the ADT, and keeps the PMP and + * PMS_SRAM power domains on; pmp-core.rs then starts the PMP and + * pmp-display.c holds the display chain and seeds the DISP request; + * - it enables the five display nodes. + * Everything is one changeset: if any check fails nothing changes and the + * Mac keeps the boot framebuffer. apple_t6031_display.pmp=0 enables the + * display nodes without adding the PMP; the display driver then adopts the + * DCP without a PMP acknowledgement (an experiment, see dcp.c). + * + * Without the option the handoff is only described in the log. + */ +#define pr_fmt(fmt) "apple-t6031-display: " fmt + +#include +#include +#include +#include +#include +#include +#include + +/* The T6031 PMP image, as the J516C ADT names it (iop-pmp-nub). */ +#define T6031_PMP_UUID "07FF4C5B-CB2E-3DE6-95FF-3B107D19B7AB" + +#define T6031_HANDOFF "apple,t6031-handoff" +#define T6031_PMP_PATH "/soc/pmp@290700000" + +static bool t6031_display_enable __initdata; +static bool t6031_display_pmp __initdata = true; + +static int __init t6031_display_enable_param(char *arg) +{ + return kstrtobool(arg, &t6031_display_enable); +} +early_param("apple_t6031_display.enable", t6031_display_enable_param); + +static int __init t6031_display_pmp_param(char *arg) +{ + return kstrtobool(arg, &t6031_display_pmp); +} +early_param("apple_t6031_display.pmp", t6031_display_pmp_param); + +/* The only node compatible with @compat, or NULL when there is none or several. */ +static struct device_node *__init t6031_unique(const char *compat) +{ + struct device_node *np, *found = NULL; + + for_each_compatible_node(np, NULL, compat) { + if (found) { + of_node_put(np); + of_node_put(found); + return NULL; + } + found = of_node_get(np); + } + return found; +} + +/* The only T6031 power state labelled @label, in any PMGR block. */ +static struct device_node *__init t6031_pwrstate(const char *label) +{ + struct device_node *np, *found = NULL; + const char *name; + + for_each_compatible_node(np, NULL, "apple,t6031-pmgr-pwrstate") { + if (of_property_read_string(np, "label", &name) || strcmp(name, label)) + continue; + if (found) { + of_node_put(np); + of_node_put(found); + return NULL; + } + found = of_node_get(np); + } + return found; +} + +static bool __init t6031_marked(const struct device_node *np) +{ + u32 val; + + return np && !of_property_read_u32(np, T6031_HANDOFF, &val) && val == 1; +} + +static bool __init t6031_disabled(const struct device_node *np) +{ + const char *status; + + return np && !of_property_read_string(np, "status", &status) && + !strcmp(status, "disabled"); +} + +/* + * The node a one-entry phandle list of @np names, if its argument cells are + * @nargs with the first equal to @arg0 and it is compatible with @compat. + */ +static struct device_node *__init t6031_linked(const struct device_node *np, + const char *list, const char *cells, + int nargs, u32 arg0, const char *compat) +{ + struct of_phandle_args args; + + if (of_count_phandle_with_args(np, list, cells) != 1 || + of_parse_phandle_with_args(np, list, cells, 0, &args)) + return NULL; + if (args.args_count != nargs || (nargs && args.args[0] != arg0) || + !of_device_is_compatible(args.np, compat)) { + of_node_put(args.np); + return NULL; + } + return args.np; +} + +static phandle __init t6031_free_phandle(void) +{ + struct device_node *np; + phandle max = 0; + + for_each_of_allnodes(np) + if (np->phandle > max) + max = np->phandle; + return max + 1; +} + +/* A property copied byte for byte; the live tree keeps it. */ +static int __init t6031_copy_prop(struct of_changeset *cs, struct device_node *np, + const struct property *src) +{ + struct property *prop = kzalloc_obj(*prop); + + if (!prop) + return -ENOMEM; + prop->name = kstrdup(src->name, GFP_KERNEL); + prop->value = kmemdup(src->value, src->length ?: 1, GFP_KERNEL); + prop->length = src->length; + if (!prop->name || !prop->value) + return -ENOMEM; + return of_changeset_add_property(cs, np, prop); +} + +/* The boot loader's values the PMP node needs: ids, capacity and tunables. */ +static bool __init t6031_pmp_value(const struct property *prop) +{ + if (!strcmp(prop->name, "apple,board-id") || + !strcmp(prop->name, "apple,dram-vendor-id") || + !strcmp(prop->name, "apple,dram-capacity")) + return prop->length == sizeof(u32); + return strstarts(prop->name, "apple,tunable-"); +} + +/* + * The PMP, its DART and its mailbox, with the J516C ADT's addresses and + * interrupts: SRAM 0x2_9070_0000 (1 MiB), ASC 0x2_90c0_0000, mailbox at + * ASC + 0x8000 (IRQs 1180-1183), DART 0x2_9030_0000 (IRQ 1184). Returns the + * number of values copied from the boot loader, or an error. + */ +static int __init t6031_add_pmp(struct of_changeset *cs, struct device_node *soc, + struct device_node *values, u32 irq_parent, + struct device_node **nodes, phandle first) +{ + static const char *const dart_compat[] = { "apple,t6031-dart", "apple,t8110-dart" }; + static const char *const mbox_compat[] = { "apple,t6031-asc-mailbox", + "apple,asc-mailbox-v4" }; + static const char *const mbox_irq_names[] = { "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty" }; + static const char *const pmp_compat[] = { "apple,t6031-pmp-v2", "apple,t6000-pmp-v2" }; + static const char *const pmp_reg_names[] = { "pmp", "asc" }; + const u32 dart_reg[] = { 0x2, 0x90300000, 0x0, 0x4000 }; + const u32 dart_irq[] = { 0, 0, 1184, 4 }; + const u32 mbox_reg[] = { 0x2, 0x90c08000, 0x0, 0x4000 }; + const u32 mbox_irq[] = { 0, 0, 1180, 4, 0, 0, 1181, 4, 0, 0, 1182, 4, 0, 0, 1183, 4 }; + const u32 pmp_reg[] = { 0x2, 0x90700000, 0x0, 0x100000, 0x2, 0x90c00000, 0x0, 0x4000 }; + const u32 iommus[] = { first, 0 }; + struct device_node *dart, *mbox, *pmp; + struct property *src; + int ret, copied = 0; + + dart = of_changeset_create_node(cs, soc, "iommu@290300000"); + mbox = of_changeset_create_node(cs, soc, "mailbox@290c08000"); + pmp = of_changeset_create_node(cs, soc, "pmp@290700000"); + if (!dart || !mbox || !pmp) + return -ENOMEM; + nodes[0] = dart; + nodes[1] = mbox; + nodes[2] = pmp; + + ret = of_changeset_add_prop_string_array(cs, dart, "compatible", dart_compat, + ARRAY_SIZE(dart_compat)) ?: + of_changeset_add_prop_u32_array(cs, dart, "reg", dart_reg, ARRAY_SIZE(dart_reg)) ?: + of_changeset_add_prop_u32(cs, dart, "#iommu-cells", 1) ?: + of_changeset_add_prop_u32(cs, dart, "interrupt-parent", irq_parent) ?: + of_changeset_add_prop_u32_array(cs, dart, "interrupts", dart_irq, + ARRAY_SIZE(dart_irq)) ?: + of_changeset_add_prop_u32(cs, dart, "phandle", first); + if (ret) + return ret; + ret = of_changeset_add_prop_string_array(cs, mbox, "compatible", mbox_compat, + ARRAY_SIZE(mbox_compat)) ?: + of_changeset_add_prop_u32_array(cs, mbox, "reg", mbox_reg, ARRAY_SIZE(mbox_reg)) ?: + of_changeset_add_prop_u32(cs, mbox, "interrupt-parent", irq_parent) ?: + of_changeset_add_prop_u32_array(cs, mbox, "interrupts", mbox_irq, + ARRAY_SIZE(mbox_irq)) ?: + of_changeset_add_prop_string_array(cs, mbox, "interrupt-names", mbox_irq_names, + ARRAY_SIZE(mbox_irq_names)) ?: + of_changeset_add_prop_u32(cs, mbox, "#mbox-cells", 0) ?: + of_changeset_add_prop_u32(cs, mbox, "phandle", first + 1); + if (ret) + return ret; + ret = of_changeset_add_prop_string_array(cs, pmp, "compatible", pmp_compat, + ARRAY_SIZE(pmp_compat)) ?: + of_changeset_add_prop_u32_array(cs, pmp, "reg", pmp_reg, ARRAY_SIZE(pmp_reg)) ?: + of_changeset_add_prop_string_array(cs, pmp, "reg-names", pmp_reg_names, + ARRAY_SIZE(pmp_reg_names)) ?: + of_changeset_add_prop_u32_array(cs, pmp, "iommus", iommus, ARRAY_SIZE(iommus)) ?: + of_changeset_add_prop_u32(cs, pmp, "mboxes", first + 1) ?: + of_changeset_add_prop_string(cs, pmp, "mbox-names", "mbox") ?: + of_changeset_add_prop_u32(cs, pmp, "phandle", first + 2); + if (ret) + return ret; + for_each_property_of_node(values, src) { + if (!t6031_pmp_value(src)) + continue; + ret = t6031_copy_prop(cs, pmp, src); + if (ret) + return ret; + copied++; + } + return copied; +} + +static int __init t6031_display_handoff(void) +{ + struct device_node *dcp = NULL, *display = NULL, *piodma = NULL; + struct device_node *mbox = NULL, *dcp_dart = NULL, *disp_dart = NULL; + struct device_node *soc = NULL, *aic = NULL, *values = NULL, *existing; + struct device_node *ps_pmp = NULL, *ps_sram = NULL, *ps, *parent; + struct device_node *added[3] = {}; + struct device_node *enable[5]; + struct of_changeset *cs = NULL; + const char *uuid = NULL, *pmp_uuid = NULL; + u32 compat[3] = {}, cells, v; + phandle first = 0; + int ret = -ENODEV, regions, i, copied = 0; + + if (!apple_pmp_t6031_board()) + return 0; + + dcp = t6031_unique("apple,t6031-dcp"); + display = t6031_unique("apple,t6031-display-subsystem"); + piodma = dcp ? of_get_child_by_name(dcp, "piodma") : NULL; + if (!dcp || !display) { + pr_info("no T6031 display description in the device tree\n"); + goto out; + } + if (!t6031_marked(dcp) || !t6031_marked(display) || !t6031_marked(piodma)) { + pr_info("the boot loader did not hand the display over (%s missing)\n", + T6031_HANDOFF); + goto out; + } + of_property_read_string(dcp, "apple,firmware-uuid", &uuid); + of_property_read_u32_array(dcp, "apple,firmware-compat", compat, 3); + regions = of_count_phandle_with_args(dcp, "memory-region", NULL); + pr_info("handoff: DCP firmware %u.%u.%u %s, %d memory regions\n", + compat[0], compat[1], compat[2], uuid ?: "(no uuid)", regions); + + if (!t6031_display_enable) { + pr_info("native display not requested; add apple_t6031_display.enable=1 to take it over, the boot framebuffer stays\n"); + ret = 0; + goto out; + } + if (compat[0] != 14 || compat[1] != 7 || compat[2] != 0 || regions < 1 || + of_count_phandle_with_args(display, "memory-region", NULL) < 1) { + pr_warn("not enabling: the display driver knows DCP firmware 14.7.0 with memory regions only\n"); + ret = -EINVAL; + goto out; + } + + mbox = t6031_linked(dcp, "mboxes", "#mbox-cells", 0, 0, "apple,asc-mailbox-v4"); + dcp_dart = t6031_linked(dcp, "iommus", "#iommu-cells", 1, 5, "apple,t8110-dart"); + disp_dart = t6031_linked(display, "iommus", "#iommu-cells", 1, 0, "apple,t8110-dart"); + if (!mbox || !dcp_dart || !disp_dart || dcp_dart == disp_dart) { + pr_warn("not enabling: the DCP's mailbox and the two display DARTs are not as expected\n"); + ret = -EINVAL; + goto out; + } + enable[0] = dcp_dart; + enable[1] = disp_dart; + enable[2] = mbox; + enable[3] = dcp; + enable[4] = display; + for (i = 0; i < ARRAY_SIZE(enable); i++) { + if (!t6031_disabled(enable[i])) { + pr_warn("not enabling: %pOF is not disabled\n", enable[i]); + ret = -EBUSY; + goto out; + } + } + + /* The display's power chain must be the boot loader's, left on. */ + ps = t6031_linked(dcp, "power-domains", "#power-domain-cells", 0, 0, + "apple,t6031-pmgr-pwrstate"); + for (i = 0; ps && i < 3; i++) { + if (!of_property_read_bool(ps, "apple,inherited-on")) { + pr_warn("not enabling: %pOF is not apple,inherited-on\n", ps); + of_node_put(ps); + ret = -EINVAL; + goto out; + } + parent = t6031_linked(ps, "power-domains", "#power-domain-cells", 0, 0, + "apple,t6031-pmgr-pwrstate"); + of_node_put(ps); + ps = parent; + } + of_node_put(ps); + + cs = kzalloc_obj(*cs); + if (!cs) { + ret = -ENOMEM; + goto out; + } + of_changeset_init(cs); + + if (t6031_display_pmp) { + values = of_find_node_by_path("/chosen/asahi,t6031-pmp"); + if (!values || + of_property_read_u32(values, "apple,board-id", &v) || + of_property_read_u32(values, "apple,dram-vendor-id", &v) || + of_property_read_string(values, "apple,tunable-uuid", &pmp_uuid)) { + pr_warn("not enabling: the boot loader passed no PMP values for this Mac\n"); + ret = -ENODEV; + goto out; + } + if (strcmp(pmp_uuid, T6031_PMP_UUID)) { + pr_warn("not enabling: this Mac's PMP firmware is %s, not %s\n", + pmp_uuid, T6031_PMP_UUID); + ret = -EINVAL; + goto out; + } + existing = of_find_node_by_path(T6031_PMP_PATH); + if (!existing) + existing = of_find_compatible_node(NULL, NULL, "apple,t6000-pmp-v2"); + if (existing) { + pr_warn("not enabling: %pOF already describes the PMP\n", existing); + of_node_put(existing); + ret = -EEXIST; + goto out; + } + soc = of_find_node_by_path("/soc"); + aic = t6031_unique("apple,t6031-aic3"); + if (!soc || !aic || !aic->phandle || + of_property_read_u32(aic, "#interrupt-cells", &cells) || cells != 4) { + pr_warn("not enabling: no /soc or no four-cell T6031 interrupt controller\n"); + ret = -EINVAL; + goto out; + } + ps_pmp = t6031_pwrstate("pmp"); + ps_sram = t6031_pwrstate("pms_sram"); + if (!ps_pmp || !ps_sram) { + pr_warn("not enabling: no unique pmp and pms_sram power states\n"); + ret = -ENODEV; + goto out; + } + first = t6031_free_phandle(); + ret = t6031_add_pmp(cs, soc, values, aic->phandle, added, first); + if (ret < 0) + goto out; + copied = ret; + /* Linux would otherwise power these off as unused. */ + ret = 0; + if (!of_property_read_bool(ps_pmp, "apple,always-on")) + ret = of_changeset_add_prop_bool(cs, ps_pmp, "apple,always-on"); + if (!ret && !of_property_read_bool(ps_sram, "apple,always-on")) + ret = of_changeset_add_prop_bool(cs, ps_sram, "apple,always-on"); + if (ret) + goto out; + } + + for (i = 0; i < ARRAY_SIZE(enable); i++) { + ret = of_changeset_update_prop_string(cs, enable[i], "status", "okay"); + if (ret) + goto out; + } + ret = of_changeset_apply(cs); + if (ret) + goto out; + /* + * A node attached by the changeset takes its phandle from the + * properties it has at that moment, before the changeset adds them. + */ + if (added[0]) { + added[0]->phandle = first; + added[1]->phandle = first + 1; + added[2]->phandle = first + 2; + } + pr_info("enabled %pOF, %pOF, %pOF, %pOF and %pOF\n", enable[0], enable[1], + enable[2], enable[3], enable[4]); + if (added[0]) + pr_info("added the PMP, its DART and mailbox with %d of this Mac's values; pmp and pms_sram kept on\n", + copied); + else + pr_info("no PMP added (apple_t6031_display.pmp=0): the DCP is adopted without a PMP acknowledgement\n"); + cs = NULL; +out: + if (ret && cs) { + /* Nothing was applied; the changeset's nodes and properties are dropped. */ + of_changeset_destroy(cs); + kfree(cs); + pr_info("display stays on the boot framebuffer (%d)\n", ret); + } + of_node_put(ps_sram); + of_node_put(ps_pmp); + of_node_put(aic); + of_node_put(soc); + of_node_put(values); + of_node_put(disp_dart); + of_node_put(dcp_dart); + of_node_put(mbox); + of_node_put(piodma); + of_node_put(display); + of_node_put(dcp); + return 0; +} + +/* Before of_platform_default_populate() at arch_initcall_sync. */ +arch_initcall(t6031_display_handoff); diff --git a/include/linux/soc/apple/pmp-display.h b/include/linux/soc/apple/pmp-display.h index 31a609ede0053..6348e2e214757 100644 --- a/include/linux/soc/apple/pmp-display.h +++ b/include/linux/soc/apple/pmp-display.h @@ -17,6 +17,18 @@ static inline bool apple_pmp_t6030_board(void) of_machine_is_compatible("apple,j516s")); } +/* + * The T6031 (M3 Max) MacBook Pros, 14-inch J514C and 16-inch J516C. Their + * PMP values come from the boot loader (/chosen/asahi,t6031-pmp); the GPU + * needs no PMP vote on this SoC. + */ +static inline bool apple_pmp_t6031_board(void) +{ + return of_machine_is_compatible("apple,t6031") && + (of_machine_is_compatible("apple,j514c") || + of_machine_is_compatible("apple,j516c")); +} + struct device; struct apple_pmp_display; @@ -36,16 +48,19 @@ void apple_pmp_display_fail(struct apple_pmp_display *display, int error); struct apple_pmp_display *apple_pmp_display_bound_owner(struct device *dev); /* - * J514S (T6030) power. pre_run holds the inherited display and seeds the PTD - * DISP|ANS request before the PMP CPU runs (called by pmp-core.rs; a no-op on - * other boards). display_prepare links the DCP to the bound PMP; get/check - * require the PMP to have acknowledged the DISP request. The GPU variants - * hold the AGX vote, retained until reboot; prepare is called during probe. + * T6030 and T6031 display power. pre_run holds the inherited display and + * seeds the PTD DISP|ANS request before the PMP CPU runs (called by + * pmp-core.rs; a no-op on other boards). display_prepare links the DCP to + * the bound PMP; get/check require the PMP to have acknowledged the DISP + * request. The J514S GPU variants hold the AGX vote, retained until reboot; + * prepare is called during probe. */ int apple_pmp_display_pre_run(struct device *dev); -int apple_pmp_j514s_display_prepare(struct device *consumer); -int apple_pmp_j514s_display_get(struct device *consumer); -int apple_pmp_j514s_display_check(struct device *consumer); +/* True once the board's PMP node exists (added by the boot-time handoff). */ +bool apple_pmp_display_present(void); +int apple_pmp_display_prepare(struct device *consumer); +int apple_pmp_display_get(struct device *consumer); +int apple_pmp_display_check(struct device *consumer); int apple_pmp_j514s_gpu_prepare(struct device *consumer); int apple_pmp_j514s_gpu_get(struct device *consumer); int apple_pmp_j514s_gpu_check(struct device *consumer);