From d77c55666a808d5c200a7e0e7a8c334d528a499f Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Thu, 22 Oct 2026 23:39:05 +0530 Subject: [PATCH 01/11] drm/apple: give the J516C panel an EDID The DCP gives no EDID for a built-in panel, so on the 14.7 panel role eDP-1 has an empty edid property and compositors see no make, model or physical size. Add a builder for a two-block EDID from the panel's device-tree node and a table of the known panel models: - the base block (EDID 1.4) names the panel by the PNP ID of the vendor prefix and the model of its most specific compatible ("apple,panel-j516" gives APP and the monitor name J516), gives its size, the model year (2023 for the J516), a nominal gamma of 2.2 and P3 primaries with a D65 white point (the panel is a P3 wide-colour panel, and the sRGB transfer function of Display P3 approximates a 2.2 gamma); - its first descriptor is a detailed timing, as EDID 1.4 requires, with the panel's geometry at 60 Hz: the same totals and horizontal porches as the panel's mode, the vertical front porch at 63 lines. The panel's own timing, 3456x2234 at 120 Hz, does not fit: the detailed timing's pixel clock field is 16 bits in 10 kHz units (at most 655.35 MHz) and the panel needs 3536 x 2456 x 120 = 1,042,129,920 Hz; its 6-bit vertical front porch cannot hold the panel's 118 lines either; - a DisplayID 2.0 extension carries the real 120 Hz timing as its preferred Type VII timing (1042130 kHz, the DCP mode line with the notch rows shown: 3456 3464 3496 3536 / 2234 2352 2360 2456), with the display parameters (size, native pixels, colour, gamma). Neither block is a mode source: DRM modes still come from the DCP firmware, and the update adds no mode. There is no product code, serial number, luminance or range descriptor. edid-decode and libdisplay-info both report the EDID conformant. dcp_panel_edid_update() publishes the EDID on a connector. Add a base_edid flag to the 14.x board records, set only in the J516C record. On that board, read the make and product from the panel node's first compatible at bind, and when a probe has modes and no other EDID, publish the panel's EDID sized as the connector is (dcp_update_connector_dimensions(): the firmware's size, else the node's), then give the connector back its size in millimetres, which the EDID update replaces with whole centimetres, and clear the firmware-derived EDID flag. Displays with a firmware EDID, the J514C (whose record carries J514S panel values) and the T6030 panels (J514S, J516S) leave the flag clear and are unchanged. On the J516C the connector keeps its one firmware mode, 3456x2160 at 120 Hz. KUnit (apple-dcp-panel-edid): the EDID is valid to drm_edid_valid(), drm_edid_get_product_id() and drm_edid_get_monitor_name() give APP, no product code or serial and J516; the base block holds 35x22 cm, gamma 2.2, model year 2023, the P3/D65 chromaticity, the 60 Hz detailed timing field by field and one extension; the DisplayID block holds its header, display parameters and every Type VII field of the 120 Hz timing, and both checksums; an update publishes 256 bytes and adds no mode. The fixture's connector uses the atomic state helpers. apple-dcp-v14-board checks that only the J516C record sets the flag (J514C, T6030 and J613 do not). Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/apple/Makefile | 3 +- drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp-panel-edid-test.c | 276 ++++++++++++++ drivers/gpu/drm/apple/dcp-panel-edid.c | 354 ++++++++++++++++++ drivers/gpu/drm/apple/dcp-panel-edid.h | 37 ++ drivers/gpu/drm/apple/dcp.c | 6 + drivers/gpu/drm/apple/iomfb.c | 16 + drivers/gpu/drm/apple/iomfb_v14_7.c | 6 + drivers/gpu/drm/apple/iomfb_v14_7.h | 2 + drivers/gpu/drm/apple/iomfb_v14_7_board.h | 5 + drivers/gpu/drm/apple/v14-board-test.c | 5 + .../selftests/drivers/gpu/apple-dcp/rate.py | 9 +- 12 files changed, 719 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-panel-edid-test.c create mode 100644 drivers/gpu/drm/apple/dcp-panel-edid.c create mode 100644 drivers/gpu/drm/apple/dcp-panel-edid.h diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index b796645b471d80..98c95bf006a127 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -7,7 +7,7 @@ appledrm-y += afk.o dcp.o dcp-fabric.o dcp_backlight.o dptxep.o iomfb.o parser.o appledrm-y += dcp-hdmi.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o -appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o +appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o dcp-panel-edid.o appledrm-y += ibootep.o appledrm-y += dcpext_scanout.o appledrm-y += dcpext_drm.o @@ -25,6 +25,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_dropped_swap_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-panel-edid-test.o appledrm-y += iomfb_v26_6.o iomfb_v26_6_link.o j613_dcp_activate.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index d22b9a638169c9..651af197e08315 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -24,6 +24,7 @@ #include "dcp-announce.h" #include "dcp-display-metadata.h" #include "dcp-metadata-epoch.h" +#include "dcp-panel-edid.h" #include "dcp-180hz.h" #include "iomfb.h" #include "iomfb_h17p.h" @@ -174,6 +175,8 @@ struct dcp_panel { int height_mm; /// panel has a mini-LED backlight bool has_mini_led; + /// make and product for the base EDID, from the panel's compatible + struct dcp_panel_id id; }; enum dcp_iomfb_method_profile { diff --git a/drivers/gpu/drm/apple/dcp-panel-edid-test.c b/drivers/gpu/drm/apple/dcp-panel-edid-test.c new file mode 100644 index 00000000000000..399f6e5bfd6470 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-panel-edid-test.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +/* + * The built-in panel's EDID, read back with the DRM EDID helpers: the make, + * product, model year and native timing come from the panel's compatible and + * the model table; the base block holds the panel's geometry at 60 Hz with P3 + * colour and gamma 2.2, the DisplayID extension the 120 Hz timing; an update + * publishes both blocks on the connector and adds no mode. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "dcp-panel-edid.h" + +#define J516_W_MM 346 +#define J516_H_MM 216 + +static void panel_id_j516(struct kunit *test, struct dcp_panel_id *id) +{ + KUNIT_ASSERT_EQ(test, dcp_panel_id_from_compatible(id, "apple,panel-j516"), 0); +} + +static void panel_id_from_compatible(struct kunit *test) +{ + struct dcp_panel_id id; + + panel_id_j516(test, &id); + KUNIT_EXPECT_STREQ(test, id.vendor, "APP"); + KUNIT_EXPECT_STREQ(test, id.name, "J516"); + KUNIT_EXPECT_EQ(test, id.model_year, 2023); + KUNIT_EXPECT_EQ(test, id.timing.clock_khz, 1042130); + KUNIT_EXPECT_EQ(test, id.timing.vtotal, 2456); + /* A model without a known year and timing gets none, and no EDID. */ + KUNIT_ASSERT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel-j999"), 0); + KUNIT_EXPECT_EQ(test, id.model_year, 0); + KUNIT_EXPECT_EQ(test, id.timing.clock_khz, 0); +} + +static void panel_id_needs_a_model_and_a_known_vendor(struct kunit *test) +{ + struct dcp_panel_id id; + + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel-"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "panel-j516"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "acme,panel-j516"), -ENOENT); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel-j516 x"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, + "apple,panel-j5160000000000"), + -EINVAL); +} + +static const struct drm_edid *panel_edid_j516(struct kunit *test, + u8 e[DCP_PANEL_EDID_BLOCKS * EDID_LENGTH]) +{ + struct dcp_panel_id id; + const struct drm_edid *edid; + + panel_id_j516(test, &id); + KUNIT_ASSERT_EQ(test, dcp_panel_edid_build(&id, J516_W_MM, J516_H_MM, e), 0); + edid = drm_edid_alloc(e, DCP_PANEL_EDID_BLOCKS * EDID_LENGTH); + KUNIT_ASSERT_NOT_NULL(test, edid); + return edid; +} + +static u8 sum(const u8 *p, size_t len) +{ + u8 s = 0; + + while (len--) + s += *p++; + return s; +} + +static void panel_edid_parses_with_make_product_and_size(struct kunit *test) +{ + struct drm_edid_product_id product; + u8 e[DCP_PANEL_EDID_BLOCKS * EDID_LENGTH]; + const struct drm_edid *edid = panel_edid_j516(test, e); + const struct edid *raw; + char vendor[4], name[14]; + + KUNIT_EXPECT_TRUE(test, drm_edid_valid(edid)); + drm_edid_get_product_id(edid, &product); + drm_edid_decode_mfg_id(be16_to_cpu(product.manufacturer_name), vendor); + KUNIT_EXPECT_STREQ(test, vendor, "APP"); + KUNIT_EXPECT_EQ(test, le16_to_cpu(product.product_code), 0); + KUNIT_EXPECT_EQ(test, le32_to_cpu(product.serial_number), 0); + + raw = drm_edid_raw(edid); + KUNIT_ASSERT_NOT_NULL(test, raw); + drm_edid_get_monitor_name(raw, name, sizeof(name)); + KUNIT_EXPECT_STREQ(test, name, "J516"); + /* The connector's size (the firmware's), in whole centimetres. */ + KUNIT_EXPECT_EQ(test, raw->width_cm, 35); + KUNIT_EXPECT_EQ(test, raw->height_cm, 22); + /* Gamma 2.2, not 1.00 (0) and not "in an extension" (0xff). */ + KUNIT_EXPECT_EQ(test, raw->gamma, 0x78); + /* Model year 2023: week 0xff, year 2023 - 1990. */ + KUNIT_EXPECT_EQ(test, raw->mfg_week, 0xff); + KUNIT_EXPECT_EQ(test, raw->mfg_year, 33); + /* P3 red (0.680, 0.320) and the D65 white point (0.3127, 0.3290), 10-bit. */ + KUNIT_EXPECT_EQ(test, raw->red_x << 2 | raw->red_green_lo >> 6, 696); + KUNIT_EXPECT_EQ(test, raw->red_y << 2 | (raw->red_green_lo >> 4 & 3), 328); + KUNIT_EXPECT_EQ(test, raw->white_x << 2 | (raw->blue_white_lo >> 2 & 3), 320); + KUNIT_EXPECT_EQ(test, raw->white_y << 2 | (raw->blue_white_lo & 3), 337); + /* One extension; both blocks sum to 0. */ + KUNIT_EXPECT_EQ(test, raw->extensions, 1); + KUNIT_EXPECT_EQ(test, sum(e, EDID_LENGTH), 0); + KUNIT_EXPECT_EQ(test, sum(e + EDID_LENGTH, EDID_LENGTH), 0); + drm_edid_free(edid); +} + +/* + * The base block's first descriptor is the panel's geometry at 60 Hz (its 120 Hz + * clock does not fit the 16-bit field); the DisplayID extension holds the real + * 120 Hz timing as its preferred Type VII timing. + */ +static void panel_edid_timings_are_the_panels(struct kunit *test) +{ + u8 e[DCP_PANEL_EDID_BLOCKS * EDID_LENGTH]; + const struct drm_edid *edid = panel_edid_j516(test, e); + const struct edid *raw = drm_edid_raw(edid); + const struct detailed_timing *d = &raw->detailed_timings[0]; + const struct detailed_pixel_timing *pt = &d->data.pixel_data; + const u8 *ext = e + EDID_LENGTH, *section = ext + 1, *t; + unsigned int i, names = 0; + + /* 3536 x 2456 x 60 Hz = 521.06 MHz, in 10 kHz. */ + KUNIT_EXPECT_EQ(test, le16_to_cpu(d->pixel_clock), 52106); + KUNIT_EXPECT_EQ(test, pt->hactive_lo | (pt->hactive_hblank_hi >> 4) << 8, 3456); + KUNIT_EXPECT_EQ(test, pt->hblank_lo | (pt->hactive_hblank_hi & 0xf) << 8, 80); + KUNIT_EXPECT_EQ(test, pt->vactive_lo | (pt->vactive_vblank_hi >> 4) << 8, 2234); + KUNIT_EXPECT_EQ(test, pt->vblank_lo | (pt->vactive_vblank_hi & 0xf) << 8, 222); + KUNIT_EXPECT_EQ(test, pt->hsync_offset_lo, 8); + KUNIT_EXPECT_EQ(test, pt->hsync_pulse_width_lo, 32); + /* The front porch at the field's 63, the sync width the panel's 8. */ + KUNIT_EXPECT_EQ(test, (pt->vsync_offset_pulse_width_lo >> 4) | + (pt->hsync_vsync_offset_pulse_width_hi >> 2 & 3) << 4, 63); + KUNIT_EXPECT_EQ(test, pt->vsync_offset_pulse_width_lo & 0xf, 8); + KUNIT_EXPECT_EQ(test, pt->width_mm_lo | (pt->width_height_mm_hi >> 4) << 8, J516_W_MM); + KUNIT_EXPECT_EQ(test, pt->height_mm_lo | (pt->width_height_mm_hi & 0xf) << 8, J516_H_MM); + /* It is not the preferred mode with its refresh rate. */ + KUNIT_EXPECT_EQ(test, raw->features & DRM_EDID_FEATURE_PREFERRED_TIMING, 0); + for (i = 1; i < ARRAY_SIZE(raw->detailed_timings); i++) { + const struct detailed_timing *o = &raw->detailed_timings[i]; + + KUNIT_EXPECT_EQ(test, le16_to_cpu(o->pixel_clock), 0); + KUNIT_EXPECT_NE(test, o->data.other_data.type, EDID_DETAIL_MONITOR_SERIAL); + KUNIT_EXPECT_NE(test, o->data.other_data.type, EDID_DETAIL_MONITOR_RANGE); + names += o->data.other_data.type == EDID_DETAIL_MONITOR_NAME; + } + KUNIT_EXPECT_EQ(test, names, 1); + for (i = 0; i < sizeof(raw->established_timings); i++) + KUNIT_EXPECT_EQ(test, ((const u8 *)&raw->established_timings)[i], 0); + + /* DisplayID 2.0, generic display, Display Parameters then one Type VII timing. */ + KUNIT_EXPECT_EQ(test, ext[0], 0x70); + KUNIT_EXPECT_EQ(test, section[0], 0x20); + KUNIT_EXPECT_EQ(test, section[2], 0x02); + KUNIT_ASSERT_EQ(test, section[1], 3 + 29 + 3 + 20); + KUNIT_EXPECT_EQ(test, sum(section, 4 + section[1] + 1), 0); + KUNIT_EXPECT_EQ(test, section[4], 0x21); + KUNIT_EXPECT_EQ(test, section[6], 29); + /* Image size in 0.1 mm, the native pixels, gamma 2.2. */ + KUNIT_EXPECT_EQ(test, section[7] | section[8] << 8, J516_W_MM * 10); + KUNIT_EXPECT_EQ(test, section[9] | section[10] << 8, J516_H_MM * 10); + KUNIT_EXPECT_EQ(test, section[11] | section[12] << 8, 3456); + KUNIT_EXPECT_EQ(test, section[13] | section[14] << 8, 2234); + KUNIT_EXPECT_EQ(test, section[7 + 28], 0x78); + t = section + 4 + 3 + 29; + KUNIT_EXPECT_EQ(test, t[0], 0x22); + KUNIT_EXPECT_EQ(test, t[2], 20); + t += 3; + /* 1042130 kHz, preferred, and the DCP mode line's porches (each minus one). */ + KUNIT_EXPECT_EQ(test, t[0] | t[1] << 8 | t[2] << 16, 1042130 - 1); + KUNIT_EXPECT_EQ(test, t[3] & 0x80, 0x80); + KUNIT_EXPECT_EQ(test, t[4] | t[5] << 8, 3456 - 1); + KUNIT_EXPECT_EQ(test, t[6] | t[7] << 8, 3536 - 3456 - 1); + KUNIT_EXPECT_EQ(test, (t[8] | t[9] << 8) & 0x7fff, 3464 - 3456 - 1); + KUNIT_EXPECT_EQ(test, t[10] | t[11] << 8, 3496 - 3464 - 1); + KUNIT_EXPECT_EQ(test, t[12] | t[13] << 8, 2234 - 1); + KUNIT_EXPECT_EQ(test, t[14] | t[15] << 8, 2456 - 2234 - 1); + KUNIT_EXPECT_EQ(test, (t[16] | t[17] << 8) & 0x7fff, 2352 - 2234 - 1); + KUNIT_EXPECT_EQ(test, t[18] | t[19] << 8, 2360 - 2352 - 1); + drm_edid_free(edid); +} + +static void panel_edid_refuses_a_size_it_cannot_hold(struct kunit *test) +{ + u8 e[DCP_PANEL_EDID_BLOCKS * EDID_LENGTH]; + struct dcp_panel_id id; + + panel_id_j516(test, &id); + KUNIT_EXPECT_EQ(test, dcp_panel_edid_build(&id, 0, J516_H_MM, e), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_edid_build(&id, J516_W_MM, 2560, e), -EINVAL); + id.model_year = 2005; + KUNIT_EXPECT_EQ(test, dcp_panel_edid_build(&id, J516_W_MM, J516_H_MM, e), -EINVAL); + id.model_year = 2023; + id.timing.clock_khz = 0; + KUNIT_EXPECT_EQ(test, dcp_panel_edid_build(&id, J516_W_MM, J516_H_MM, e), -EINVAL); + panel_id_j516(test, &id); + id.vendor[0] = '\0'; + KUNIT_EXPECT_EQ(test, dcp_panel_edid_build(&id, J516_W_MM, J516_H_MM, e), -EINVAL); +} + +struct panel_edid_test { + struct drm_device drm; + struct drm_connector connector; +}; + +/* The DRM KUnit device is atomic: a connector needs its state helpers. */ +static const struct drm_connector_funcs panel_edid_connector_funcs = { + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static void panel_edid_update_publishes_the_block(struct kunit *test) +{ + struct panel_edid_test *t; + struct drm_connector *connector; + struct dcp_panel_id id; + struct device *dev; + int ret; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + t = drm_kunit_helper_alloc_drm_device(test, dev, struct panel_edid_test, drm, + DRIVER_MODESET); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t); + connector = &t->connector; + ret = drmm_connector_init(&t->drm, connector, &panel_edid_connector_funcs, + DRM_MODE_CONNECTOR_eDP, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + + panel_id_j516(test, &id); + mutex_lock(&t->drm.mode_config.mutex); + ret = dcp_panel_edid_update(connector, &id, J516_W_MM, J516_H_MM); + mutex_unlock(&t->drm.mode_config.mutex); + KUNIT_ASSERT_EQ(test, ret, 0); + + /* The update sets 350x220 from the block; dcp_get_modes() restores the mm. */ + KUNIT_EXPECT_EQ(test, connector->display_info.width_mm, 350); + KUNIT_EXPECT_EQ(test, connector->display_info.height_mm, 220); + KUNIT_ASSERT_NOT_NULL(test, connector->edid_blob_ptr); + KUNIT_EXPECT_EQ(test, connector->edid_blob_ptr->length, 2 * EDID_LENGTH); + /* The update adds no mode: the modes stay the firmware's. */ + KUNIT_EXPECT_TRUE(test, list_empty(&connector->probed_modes)); +} + +static struct kunit_case panel_edid_cases[] = { + KUNIT_CASE(panel_id_from_compatible), + KUNIT_CASE(panel_id_needs_a_model_and_a_known_vendor), + KUNIT_CASE(panel_edid_parses_with_make_product_and_size), + KUNIT_CASE(panel_edid_timings_are_the_panels), + KUNIT_CASE(panel_edid_refuses_a_size_it_cannot_hold), + KUNIT_CASE(panel_edid_update_publishes_the_block), + {} +}; + +static struct kunit_suite panel_edid_suite = { + .name = "apple-dcp-panel-edid", + .test_cases = panel_edid_cases, +}; + +kunit_test_suite(panel_edid_suite); diff --git a/drivers/gpu/drm/apple/dcp-panel-edid.c b/drivers/gpu/drm/apple/dcp-panel-edid.c new file mode 100644 index 00000000000000..97eee161016978 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-panel-edid.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * An EDID for a built-in panel. + * + * The DCP gives no EDID for a built-in panel, so a compositor sees no make, + * model or physical size for it. This builds one from what the panel's + * device-tree node and a table of the known panel models say: the vendor and + * model of its most specific compatible ("apple,panel-j516": make APP, name + * J516), the size the connector has, the model year, the panel's native timing + * as the DCP's mode line gives it with all rows shown, and the colour of a P3 + * wide-colour panel (P3 primaries, D65 white point and a nominal gamma of + * 2.2, which the sRGB transfer function of Display P3 approximates). + * + * The native timing is 3456x2234 at 120 Hz with a pixel clock of 1042 MHz. + * The base block's detailed timing cannot hold it: its pixel clock field is + * 16 bits in 10 kHz units (at most 655.35 MHz) and its vertical front porch 6 + * bits. The base block therefore carries the panel's geometry at 60 Hz (the + * same totals and horizontal porches, the vertical front porch at most 63 + * lines), and a DisplayID 2.0 extension carries the real 120 Hz timing as its + * preferred timing, with the display parameters. Neither is a mode source: + * the driver publishes the firmware's modes and never adds modes from this + * EDID. It makes no other claim: no product code or serial number, no + * manufacture week, no luminance and no refresh range. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "dcp-panel-edid.h" + +/* Device-tree vendor prefixes and their PNP IDs. */ +static const struct { + const char *prefix; + const char *pnp; +} dcp_panel_vendors[] = { + { "apple", "APP" }, +}; + +/* + * Panel models: the year their machines came out (the EDID model year) and the + * native timing of the DCP's mode line, with the rows the notch hides shown. + */ +static const struct { + const char *name; + u16 year; + struct dcp_panel_timing timing; +} dcp_panel_models[] = { + /* MacBook Pro (16-inch, M3 Pro or M3 Max): 3456x2234 at 120 Hz */ + { "J516", 2023, { 1042130, 3456, 3464, 3496, 3536, 2234, 2352, 2360, 2456 } }, +}; + +/* EDID 1.4: a nominal gamma of 2.2 is (2.2 - 1) * 100 = 120. */ +#define DCP_PANEL_EDID_GAMMA 120 +/* EDID 1.4: week 0xff marks the year byte as the model year, from 1990. */ +#define DCP_PANEL_EDID_MODEL_YEAR 0xff +/* The base block's detailed timing: the panel's geometry at this rate. */ +#define DCP_PANEL_EDID_DTD_HZ 60 +/* EDID 1.4 detailed timing: the vertical front porch field is 6 bits. */ +#define DCP_PANEL_EDID_DTD_VFP_MAX 63 + +/* P3 primaries and the D65 white point, CIE 1931 x and y in 1/10000. */ +static const u16 dcp_panel_p3[8] = { 6800, 3200, 2650, 6900, 1500, 600, 3127, 3290 }; + +/* DisplayID 2.0 (VESA DisplayID Standard v2.0). */ +#define DISPLAYID_EXT 0x70 /* EDID extension tag */ +#define DISPLAYID_V20 0x20 /* structure version 2, revision 0 */ +#define DISPLAYID_GENERIC 0x02 /* primary use case: generic display */ +#define DISPLAYID_PARAMS 0x21 /* Display Parameters data block */ +#define DISPLAYID_PARAMS_LEN 29 +#define DISPLAYID_TYPE7 0x22 /* Type VII detailed timing data block */ +#define DISPLAYID_TYPE7_LEN 20 +#define DISPLAYID_PREFERRED 0x80 /* timing options: preferred */ +#define DISPLAYID_ASPECT_NONE 0x08 /* aspect ratio not given (from the pixels) */ +#define DISPLAYID_SYNC_POS 0x8000 /* front porch word: positive sync */ +#define DISPLAYID_NO_SPEAKERS 0x80 /* features: audio not integrated */ +#define DISPLAYID_TECH_LCD 0x10 /* device technology: active-matrix LCD */ + +int dcp_panel_id_from_compatible(struct dcp_panel_id *id, const char *compatible) +{ + const char *comma = strchr(compatible, ','), *model; + size_t vendor_len, len, i; + + memset(id, 0, sizeof(*id)); + if (!comma) + return -EINVAL; + vendor_len = comma - compatible; + model = comma + 1; + if (str_has_prefix(model, "panel-")) + model += strlen("panel-"); + else + return -EINVAL; + len = strlen(model); + if (!len || len > DCP_PANEL_NAME_LEN) + return -EINVAL; + for (i = 0; i < len; i++) { + if (!isalnum(model[i]) && model[i] != '-') + return -EINVAL; + id->name[i] = toupper(model[i]); + } + + for (i = 0; i < ARRAY_SIZE(dcp_panel_models); i++) { + if (!strcmp(id->name, dcp_panel_models[i].name)) { + id->model_year = dcp_panel_models[i].year; + id->timing = dcp_panel_models[i].timing; + } + } + for (i = 0; i < ARRAY_SIZE(dcp_panel_vendors); i++) { + if (strlen(dcp_panel_vendors[i].prefix) == vendor_len && + !strncmp(compatible, dcp_panel_vendors[i].prefix, vendor_len)) { + strscpy(id->vendor, dcp_panel_vendors[i].pnp); + return 0; + } + } + memset(id, 0, sizeof(*id)); + return -ENOENT; +} + +static void put_le16(u8 *p, u16 v) +{ + p[0] = v; + p[1] = v >> 8; +} + +static void checksum(u8 *start, size_t len) +{ + u8 sum = 0; + size_t i; + + for (i = 0; i < len - 1; i++) + sum += start[i]; + start[len - 1] = -sum; +} + +static bool timing_valid(const struct dcp_panel_timing *t) +{ + return t->clock_khz && t->hdisplay < t->hsync_start && t->hsync_start < t->hsync_end && + t->hsync_end < t->htotal && t->htotal - t->hdisplay < 4096 && + t->vdisplay < t->vsync_start && t->vsync_start < t->vsync_end && + t->vsync_end < t->vtotal && t->vtotal - t->vdisplay < 4096 && + t->hsync_start - t->hdisplay < 1024 && t->hsync_end - t->hsync_start < 1024 && + t->vsync_end - t->vsync_start < 64; +} + +/* The base block's detailed timing: the panel's geometry at DCP_PANEL_EDID_DTD_HZ. */ +static void base_dtd(u8 *d, const struct dcp_panel_timing *t, int width_mm, int height_mm) +{ + u32 clock = DIV_ROUND_CLOSEST((u32)t->htotal * t->vtotal * DCP_PANEL_EDID_DTD_HZ, 10000); + u16 hblank = t->htotal - t->hdisplay, vblank = t->vtotal - t->vdisplay; + u16 hfp = t->hsync_start - t->hdisplay, hsw = t->hsync_end - t->hsync_start; + u16 vfp = min_t(u16, t->vsync_start - t->vdisplay, DCP_PANEL_EDID_DTD_VFP_MAX); + u16 vsw = t->vsync_end - t->vsync_start; + + put_le16(d, clock); + d[2] = t->hdisplay; + d[3] = hblank; + d[4] = (t->hdisplay >> 8) << 4 | hblank >> 8; + d[5] = t->vdisplay; + d[6] = vblank; + d[7] = (t->vdisplay >> 8) << 4 | vblank >> 8; + d[8] = hfp; + d[9] = hsw; + d[10] = (vfp & 0xf) << 4 | (vsw & 0xf); + d[11] = (hfp >> 8) << 6 | (hsw >> 8) << 4 | (vfp >> 4) << 2 | (vsw >> 4); + d[12] = width_mm; + d[13] = height_mm; + d[14] = (width_mm >> 8) << 4 | height_mm >> 8; + /* Digital separate sync, positive vertical and horizontal sync. */ + d[17] = 0x1e; +} + +/* The P3/D65 chromaticity of the base block, in 10-bit fractions. */ +static void base_chromaticity(struct edid *block) +{ + u16 v[8]; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(v); i++) + v[i] = DIV_ROUND_CLOSEST(dcp_panel_p3[i] * 1024, 10000); + block->red_green_lo = (v[0] & 3) << 6 | (v[1] & 3) << 4 | (v[2] & 3) << 2 | (v[3] & 3); + block->blue_white_lo = (v[4] & 3) << 6 | (v[5] & 3) << 4 | (v[6] & 3) << 2 | (v[7] & 3); + block->red_x = v[0] >> 2; + block->red_y = v[1] >> 2; + block->green_x = v[2] >> 2; + block->green_y = v[3] >> 2; + block->blue_x = v[4] >> 2; + block->blue_y = v[5] >> 2; + block->white_x = v[6] >> 2; + block->white_y = v[7] >> 2; +} + +static int base_block(const struct dcp_panel_id *id, int width_mm, int height_mm, + struct edid *block) +{ + struct detailed_non_pixel *name; + u16 manufacturer = 0; + size_t len = strnlen(id->name, sizeof(id->name)); + unsigned int i; + + for (i = 0; i < 3; i++) { + if (id->vendor[i] < 'A' || id->vendor[i] > 'Z') + return -EINVAL; + manufacturer = manufacturer << 5 | (id->vendor[i] - '@'); + } + + memset(block, 0, sizeof(*block)); + memcpy(block->header, "\x00\xff\xff\xff\xff\xff\xff\x00", sizeof(block->header)); + block->product_id.manufacturer_name = cpu_to_be16(manufacturer); + block->mfg_week = DCP_PANEL_EDID_MODEL_YEAR; + block->mfg_year = id->model_year - 1990; + block->version = 1; + block->revision = 4; + block->input = DRM_EDID_INPUT_DIGITAL; + block->width_cm = DIV_ROUND_CLOSEST(width_mm, 10); + block->height_cm = DIV_ROUND_CLOSEST(height_mm, 10); + block->gamma = DCP_PANEL_EDID_GAMMA; + base_chromaticity(block); + /* Standard timings 0x0101 are unused slots. */ + memset(block->standard_timings, 0x01, sizeof(block->standard_timings)); + + /* EDID 1.4: the first descriptor is a detailed timing; the second names the panel. */ + base_dtd((u8 *)&block->detailed_timings[0], &id->timing, width_mm, height_mm); + name = &block->detailed_timings[1].data.other_data; + name->type = EDID_DETAIL_MONITOR_NAME; + memset(name->data.str.str, ' ', sizeof(name->data.str.str)); + memcpy(name->data.str.str, id->name, len); + if (len < sizeof(name->data.str.str)) + name->data.str.str[len] = '\n'; + for (i = 2; i < ARRAY_SIZE(block->detailed_timings); i++) + block->detailed_timings[i].data.other_data.type = 0x10; + block->extensions = DCP_PANEL_EDID_BLOCKS - 1; + checksum((u8 *)block, EDID_LENGTH); + return 0; +} + +/* DisplayID 2.0 CIE 1931 x/y pair in 12-bit fractions, packed in 3 bytes. */ +static void displayid_xy(u8 *p, u16 x, u16 y) +{ + u16 X = DIV_ROUND_CLOSEST(x * 4095, 10000), Y = DIV_ROUND_CLOSEST(y * 4095, 10000); + + p[0] = X; + p[1] = (X >> 8) | (Y & 0xf) << 4; + p[2] = Y >> 4; +} + +/* The DisplayID 2.0 extension: display parameters and the 120 Hz Type VII timing. */ +static void displayid_block(const struct dcp_panel_id *id, int width_mm, int height_mm, + u8 *ext) +{ + const struct dcp_panel_timing *t = &id->timing; + u8 *section = ext + 1, *p, *b; + unsigned int i; + + memset(ext, 0, EDID_LENGTH); + ext[0] = DISPLAYID_EXT; + section[0] = DISPLAYID_V20; + section[1] = 3 + DISPLAYID_PARAMS_LEN + 3 + DISPLAYID_TYPE7_LEN; + section[2] = DISPLAYID_GENERIC; + section[3] = 0; /* no further DisplayID extensions */ + + b = section + 4; + b[0] = DISPLAYID_PARAMS; + b[2] = DISPLAYID_PARAMS_LEN; + p = b + 3; + put_le16(p + 0, width_mm * 10); /* 0.1 mm */ + put_le16(p + 2, height_mm * 10); + put_le16(p + 4, t->hdisplay); + put_le16(p + 6, t->vdisplay); + /* Left to right, top to bottom; CIE 1931; the speakers are not the panel's. */ + p[8] = DISPLAYID_NO_SPEAKERS; + for (i = 0; i < 4; i++) + displayid_xy(p + 9 + 3 * i, dcp_panel_p3[2 * i], dcp_panel_p3[2 * i + 1]); + /* Bytes 21-26, the luminances, are not known: left 0. Colour depth not given. */ + p[27] = DISPLAYID_TECH_LCD; + p[28] = DCP_PANEL_EDID_GAMMA; + + b = p + DISPLAYID_PARAMS_LEN; + b[0] = DISPLAYID_TYPE7; + b[2] = DISPLAYID_TYPE7_LEN; + p = b + 3; + /* Pixel clock in kHz minus one, 24 bits. */ + p[0] = (t->clock_khz - 1); + p[1] = (t->clock_khz - 1) >> 8; + p[2] = (t->clock_khz - 1) >> 16; + p[3] = DISPLAYID_PREFERRED | DISPLAYID_ASPECT_NONE; + /* Each field minus one; the front porch words carry the sync polarity. */ + put_le16(p + 4, t->hdisplay - 1); + put_le16(p + 6, t->htotal - t->hdisplay - 1); + put_le16(p + 8, (t->hsync_start - t->hdisplay - 1) | DISPLAYID_SYNC_POS); + put_le16(p + 10, t->hsync_end - t->hsync_start - 1); + put_le16(p + 12, t->vdisplay - 1); + put_le16(p + 14, t->vtotal - t->vdisplay - 1); + put_le16(p + 16, (t->vsync_start - t->vdisplay - 1) | DISPLAYID_SYNC_POS); + put_le16(p + 18, t->vsync_end - t->vsync_start - 1); + + /* The section's checksum after its payload, then the block's. */ + checksum(section, 4 + section[1] + 1); + checksum(ext, EDID_LENGTH); +} + +int dcp_panel_edid_build(const struct dcp_panel_id *id, int width_mm, int height_mm, + u8 edid[DCP_PANEL_EDID_BLOCKS * EDID_LENGTH]) +{ + size_t len = strnlen(id->name, sizeof(id->name)); + int ret; + + if (width_mm <= 0 || height_mm <= 0 || width_mm > 4095 || height_mm > 4095 || + DIV_ROUND_CLOSEST(width_mm, 10) > 255 || DIV_ROUND_CLOSEST(height_mm, 10) > 255) + return -EINVAL; + if (strnlen(id->vendor, sizeof(id->vendor)) != 3 || !len || len > DCP_PANEL_NAME_LEN) + return -EINVAL; + /* EDID 1.4 reserves the years before 2006. */ + if (id->model_year < 2006 || id->model_year > 1990 + 255) + return -EINVAL; + if (!timing_valid(&id->timing)) + return -EINVAL; + ret = base_block(id, width_mm, height_mm, (struct edid *)edid); + if (ret) + return ret; + displayid_block(id, width_mm, height_mm, edid + EDID_LENGTH); + return 0; +} + +int dcp_panel_edid_update(struct drm_connector *connector, const struct dcp_panel_id *id, + int width_mm, int height_mm) +{ + const struct drm_edid *edid; + u8 *blocks; + int ret; + + blocks = kmalloc(DCP_PANEL_EDID_BLOCKS * EDID_LENGTH, GFP_KERNEL); + if (!blocks) + return -ENOMEM; + ret = dcp_panel_edid_build(id, width_mm, height_mm, blocks); + if (ret) + goto out; + edid = drm_edid_alloc(blocks, DCP_PANEL_EDID_BLOCKS * EDID_LENGTH); + if (!edid) { + ret = -ENOMEM; + goto out; + } + /* This sets the connector's size from the block, in whole centimetres. */ + ret = drm_edid_connector_update(connector, edid); + drm_edid_free(edid); +out: + kfree(blocks); + return ret; +} diff --git a/drivers/gpu/drm/apple/dcp-panel-edid.h b/drivers/gpu/drm/apple/dcp-panel-edid.h new file mode 100644 index 00000000000000..1e9a6242831b11 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-panel-edid.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* A base EDID for a built-in panel, from its device-tree node. */ +#ifndef __APPLE_DCP_PANEL_EDID_H__ +#define __APPLE_DCP_PANEL_EDID_H__ + +#include + +/* The monitor-name text of an EDID descriptor holds 13 characters. */ +#define DCP_PANEL_NAME_LEN 13 + +/* An EDID of a base block and one DisplayID extension. */ +#define DCP_PANEL_EDID_BLOCKS 2 + +/* A panel's native timing, as the DCP's mode line gives it (all rows shown). */ +struct dcp_panel_timing { + u32 clock_khz; + u16 hdisplay, hsync_start, hsync_end, htotal; + u16 vdisplay, vsync_start, vsync_end, vtotal; +}; + +/* The make, product, model year and native timing of a panel. */ +struct dcp_panel_id { + char vendor[4]; /* PNP ID, empty if unknown */ + char name[DCP_PANEL_NAME_LEN + 1]; /* monitor name */ + u16 model_year; /* the model's year, 0 if unknown */ + struct dcp_panel_timing timing; /* zero if unknown */ +}; + +struct drm_connector; + +int dcp_panel_id_from_compatible(struct dcp_panel_id *id, const char *compatible); +int dcp_panel_edid_build(const struct dcp_panel_id *id, int width_mm, int height_mm, + u8 edid[DCP_PANEL_EDID_BLOCKS * 128]); +int dcp_panel_edid_update(struct drm_connector *connector, const struct dcp_panel_id *id, + int width_mm, int height_mm); + +#endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8c877d0c2434b9..835b7293823edc 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3768,6 +3768,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat if (panel_np) { const char height_prop[2][16] = { "adj-height-mm", "height-mm" }; + const char *compatible; if (of_device_is_available(panel_np)) { ret = of_property_read_u32(panel_np, "apple,max-brightness", @@ -3780,6 +3781,11 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat /* use adjusted height as long as the notch is hidden */ of_property_read_u32(panel_np, height_prop[!dcp->notch_height], &dcp->panel.height_mm); + /* The most specific compatible names the panel for its EDID. */ + if (iomfb_v14_7_base_edid(dcp) && + !of_property_read_string_index(panel_np, "compatible", 0, &compatible) && + dcp_panel_id_from_compatible(&dcp->panel.id, compatible)) + dev_dbg(dev, "no EDID make and product in '%s'\n", compatible); of_node_put(panel_np); dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1efc58efd6f620..e505d0788388d0 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1878,6 +1878,22 @@ int dcp_get_modes(struct drm_connector *connector) } else { apple_connector->drm_edid_firmware_derived = false; } + } else if (nr_modes && dcp_is_v14_7_panel(dcp) && dcp->panel.id.vendor[0]) { + /* + * No EDID comes for this panel: publish the base block of its + * node, sized as the connector is (the firmware's size, else the + * node's), then give the connector back its size in millimetres. + */ + int update; + + dcp_update_connector_dimensions(dcp, connector); + update = dcp_panel_edid_update(connector, &dcp->panel.id, + connector->display_info.width_mm, + connector->display_info.height_mm); + dcp_update_connector_dimensions(dcp, connector); + apple_connector->drm_edid_firmware_derived = false; + if (update) + dev_warn_once(dcp->dev, "panel EDID publication failed: %d\n", update); } else { u64 generation = 0; const struct drm_edid *derived = nr_modes ? diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 474841ff8d0aec..a5484dadb0cdb6 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -159,6 +159,7 @@ static const struct dcp_v14_board dcp_v14_board_j516c = { .panel_height = 2234, .promotion = true, .log_bringup = true, + .base_edid = true, }; /* @@ -1541,6 +1542,11 @@ const char *iomfb_v14_7_board_name(const struct dcp_v14_board *board) return board->name; } +bool iomfb_v14_7_base_edid(const struct apple_dcp *dcp) +{ + return dcp->v14 && dcp->v14->board && dcp->v14->board->base_edid; +} + int iomfb_v14_7_probe(struct apple_dcp *dcp) { struct device *dev = dcp->dev; diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index f9bef6ed0f109f..be801416508af1 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -28,6 +28,8 @@ const __be32 *of_translate_dma_region_checked(struct device_node *dev, */ const struct dcp_v14_board *iomfb_v14_7_board(struct device *dev); const char *iomfb_v14_7_board_name(const struct dcp_v14_board *board); +/* Whether the internal panel's board record gives it a base EDID. */ +bool iomfb_v14_7_base_edid(const struct apple_dcp *dcp); /* Platform probe: firmware identity, boot loader handoff, PMP acknowledgment. */ int iomfb_v14_7_probe(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_board.h b/drivers/gpu/drm/apple/iomfb_v14_7_board.h index 368dab49c29445..71ec4d9754b4af 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_board.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7_board.h @@ -39,6 +39,11 @@ struct dcp_v14_board { * releases, so they are only set where tested. */ u32 ctm_set, ctm_get; + /* + * The panel gets a base EDID (make, model and size) built from its + * device-tree node, as the firmware gives none. + */ + bool base_edid; }; /* Whether the DCP node or the machine (as @ctx gives them) has @compat. */ diff --git a/drivers/gpu/drm/apple/v14-board-test.c b/drivers/gpu/drm/apple/v14-board-test.c index 801ed2f623a4c5..52ec3452af7499 100644 --- a/drivers/gpu/drm/apple/v14-board-test.c +++ b/drivers/gpu/drm/apple/v14-board-test.c @@ -71,6 +71,7 @@ static void v14_board_j516c(struct kunit *test) /* No colour matrix method is known for this board. */ KUNIT_EXPECT_EQ(test, b->ctm_set, 0); KUNIT_EXPECT_EQ(test, b->ctm_get, 0); + KUNIT_EXPECT_TRUE(test, b->base_edid); } /* @@ -95,6 +96,8 @@ static void v14_board_j514c(struct kunit *test) KUNIT_EXPECT_TRUE(test, b->promotion); KUNIT_EXPECT_TRUE(test, b->log_bringup); KUNIT_EXPECT_EQ(test, b->ctm_set, 0); + /* Its panel is not the J516C's: no base EDID. */ + KUNIT_EXPECT_FALSE(test, b->base_edid); KUNIT_EXPECT_TRUE(test, dcp_v14_board_admits(b, J516C_DCP_IMAGE)); KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); /* Not the J516C record. */ @@ -153,6 +156,7 @@ static void v14_board_unchanged(struct kunit *test) KUNIT_EXPECT_TRUE(test, b->promotion); KUNIT_EXPECT_FALSE(test, b->log_bringup); KUNIT_EXPECT_EQ(test, b->ctm_set, 0); + KUNIT_EXPECT_FALSE(test, b->base_edid); KUNIT_EXPECT_TRUE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, J613_DCP_IMAGE)); /* The T6030 record serves a T6030 DCP on any machine, as before. */ @@ -167,6 +171,7 @@ static void v14_board_unchanged(struct kunit *test) KUNIT_EXPECT_EQ(test, b->panel_height, 1664); KUNIT_EXPECT_FALSE(test, b->promotion); KUNIT_EXPECT_FALSE(test, b->log_bringup); + KUNIT_EXPECT_FALSE(test, b->base_edid); KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); /* A T8122 DCP on a T6031 machine has no record. */ diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index 5ac596625d9acc..2ab89ebfc7d5db 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -554,7 +554,7 @@ def publication(root): state = n['state'] state = state.replace('struct apple_connector {', '''struct drm_device { void *dev; struct { bool mutex; } mode_config; }; struct drm_connector { struct drm_device *dev; int registration_state; void *state; - struct { struct { unsigned min_vfreq,max_vfreq; } monitor_range; } display_info; + struct { struct { unsigned min_vfreq,max_vfreq; } monitor_range; int width_mm,height_mm; } display_info; }; struct apple_connector {''') state = state.replace(' struct { int registration_state; } base;', ' struct drm_connector base;') @@ -569,7 +569,7 @@ def publication(root): int fw_compat,width_mm,height_mm,fixed_connector_type; unsigned notch_height; bool modes_lock,modes_admitted,modes_provisional,placeholder_retried,external_native; int external_held; u64 modes_generation,dimensions_generation; struct work_struct dimensions_wq; - struct { int width_mm,height_mm; } panel; + struct { int width_mm,height_mm; struct { char vendor[4]; } id; } panel; struct { size_t length,received; void *data; bool contiguous,payload_complete; u64 modes_generation; u64 av_service_epoch,automatic_connection_sequence; bool automatic_service_pending; struct dcp_metadata_ticket metadata_ticket,mode_ticket; @@ -612,6 +612,11 @@ def publication(root): static bool cancel_work_sync(struct work_struct *w) { assert(!w->scheduled); return false; } /* Native audio is out of scope: reject entry instead of silently queueing it. */ static void dcp_external_audio_start(struct apple_dcp *d) { assert(!d->external_native); } +/* These H17P scenarios never reach the 14.7 panel's base EDID. */ +#define dev_warn_once(...) ((void)0) +static bool dcp_is_v14_7_panel(const struct apple_dcp *d) { (void)d; return false; } +static int dcp_panel_edid_update(struct drm_connector *c,const void *id,int w,int h) { (void)c;(void)id;(void)w;(void)h;assert(false);return 0; } +static void dcp_update_connector_dimensions(struct apple_dcp *d,struct drm_connector *c) { (void)d;(void)c;assert(false); } static void dpavservep_detach(struct apple_dcp *d); static struct apple_dcp *probing; static struct drm_edid transport_edid; From e3f69b773c8fa6711b41fdc3f96dbe382322b6b9 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:12:09 +0530 Subject: [PATCH 02/11] drm/apple: stamp the J516C panel's 14.7 swaps On the J516C the 120 Hz panel presented at 60 fps: the 14.7 swap record is sent with its timestamps zeroed, and the firmware then completes at most one swap per 16.7 ms. Counter values (1 GHz on T6031) in the timestamp slots make the firmware time out on the first swap. Write the constant 120 in timestamp slots 0, 1 and 6 instead, a time that is never in the future, after the swap record is encoded. Add a swap_timestamps flag to the 14.x board records, set only in the J516C record, and stamp a swap only for the internal panel of such a record. The J514C (untested), the T6030 (J514S, J516S) and T8122 panels and the external processors, which have no board record, send zero timestamps, byte for byte as before. The counter timestamps of the IOMFB template path (VRR modes on a main display) are not used by the native 14.7 path and are unchanged. Measured on the J516C on this branch: kmscube (atomic) presents at 120 fps (120.03 on two boots) on the panel's 120 Hz mode, 3456x2160 with the 74 notch rows hidden, and the first swaps of a boot complete in 2.9 to 8.1 ms instead of 16.7 ms. KUnit (apple-dcp-v14-board): the J516C record stamps its internal panel's swaps; the external role, the J514C, T6030 and J613 records and no record do not; a stamped swap holds 120 in slots 0, 1 and 6 and is otherwise the encoded record, with and without a surface. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/apple/iomfb_v14_7.c | 8 +++ drivers/gpu/drm/apple/iomfb_v14_7_board.h | 14 +++++ drivers/gpu/drm/apple/iomfb_v14_7_swap.h | 17 +++++ drivers/gpu/drm/apple/v14-board-test.c | 77 ++++++++++++++++++++++- 4 files changed, 115 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index a5484dadb0cdb6..0acafac676dcdd 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -160,6 +160,7 @@ static const struct dcp_v14_board dcp_v14_board_j516c = { .promotion = true, .log_bringup = true, .base_edid = true, + .swap_timestamps = true, }; /* @@ -1542,6 +1543,11 @@ const char *iomfb_v14_7_board_name(const struct dcp_v14_board *board) return board->name; } +bool dcp_v14_board_stamps_swaps(const struct dcp_v14_board *board, bool external) +{ + return board && !external && board->swap_timestamps; +} + bool iomfb_v14_7_base_edid(const struct apple_dcp *dcp) { return dcp->v14 && dcp->v14->board && dcp->v14->board->base_edid; @@ -2615,6 +2621,8 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, if (!ret) { dcp_v14_encode_swap(swap, id, DCP_V14_BLACK, surface, iova, width, height, dst_y, DCP_V14_LAYER); + if (dcp_v14_board_stamps_swaps(v14->board, v14->external)) + dcp_v14_swap_timestamps(swap); ret = dcp_v14_call_guarded(v14, A(407), swap, DCP_V14_SWAP_SIZE, result, sizeof(result), id, dcp_v14_swap_admit, &admission); /* The swap status sits at byte 5 of the reply. */ diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_board.h b/drivers/gpu/drm/apple/iomfb_v14_7_board.h index 71ec4d9754b4af..c677bbbafa04db 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_board.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7_board.h @@ -44,6 +44,13 @@ struct dcp_v14_board { * device-tree node, as the firmware gives none. */ bool base_edid; + /* + * The panel's swaps carry the constant timestamp + * DCP_V14_SWAP_TIMESTAMP_VALUE in slots 0, 1 and 6. With zero + * timestamps this board's firmware completes at most one swap per + * 16.7 ms, so a 120 Hz panel presents at 60 fps. + */ + bool swap_timestamps; }; /* Whether the DCP node or the machine (as @ctx gives them) has @compat. */ @@ -64,4 +71,11 @@ const struct dcp_v14_board *dcp_v14_board_select(dcp_v14_compat_fn dcp_is, */ bool dcp_v14_board_admits(const struct dcp_v14_board *board, const char *uuid); +/* + * Whether a swap carries timestamps: only the internal panel of a board whose + * record asks for them. External processors (@external, no record) and every + * other board send zero timestamps. + */ +bool dcp_v14_board_stamps_swaps(const struct dcp_v14_board *board, bool external); + #endif /* __APPLE_IOMFB_V14_7_BOARD_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_swap.h b/drivers/gpu/drm/apple/iomfb_v14_7_swap.h index aab3351dd9eea4..7fe32cabfae65c 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_swap.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7_swap.h @@ -11,6 +11,8 @@ #define DCP_V14_SURFACE_SIZE 0x22c /* Offsets in the swap record. Rectangles are x, y, w, h (u32 each). */ +#define DCP_V14_SWAP_TIMESTAMP(n) ((n) * 8) /* u64, slots 0 to 6 */ +#define DCP_V14_SWAP_TIMESTAMPS 7 #define DCP_V14_SWAP_ID 0x98 #define DCP_V14_SWAP_SURF_ID(p) (0x9c + (p) * 4) #define DCP_V14_SWAP_SRC(p) (0xac + (p) * 16) /* in the surface */ @@ -115,4 +117,19 @@ static inline void dcp_v14_encode_swap(u8 *swap, u32 id, u32 background, const u } } +/* + * The swap timestamps of a board that asks for them: the constant 120 in + * slots 0, 1 and 6, a time that is never in the future. Counter values + * (1 GHz on T6031) make the firmware time out on the first swap. Call after + * dcp_v14_encode_swap(), which zeroes the record. + */ +#define DCP_V14_SWAP_TIMESTAMP_VALUE 120 + +static inline void dcp_v14_swap_timestamps(u8 *swap) +{ + put_unaligned_le64(DCP_V14_SWAP_TIMESTAMP_VALUE, swap + DCP_V14_SWAP_TIMESTAMP(0)); + put_unaligned_le64(DCP_V14_SWAP_TIMESTAMP_VALUE, swap + DCP_V14_SWAP_TIMESTAMP(1)); + put_unaligned_le64(DCP_V14_SWAP_TIMESTAMP_VALUE, swap + DCP_V14_SWAP_TIMESTAMP(6)); +} + #endif /* __APPLE_IOMFB_V14_7_SWAP_H__ */ diff --git a/drivers/gpu/drm/apple/v14-board-test.c b/drivers/gpu/drm/apple/v14-board-test.c index 52ec3452af7499..98df7dfab36ea8 100644 --- a/drivers/gpu/drm/apple/v14-board-test.c +++ b/drivers/gpu/drm/apple/v14-board-test.c @@ -3,12 +3,16 @@ * The 14.x IOMFB board records: which record a DCP gets on which machine, * and which DCP firmware image a record admits. The J516C (T6031) record * admits only the T6030 image, whose IOMFB layouts were validated, and - * refuses every other; the existing records are unchanged. + * refuses every other; the existing records are unchanged. Which DCPs stamp + * their swaps: the internal panel of a T6031 record; the T6030 and T8122 + * panels and external processors keep zero timestamps, byte for byte. */ #include #include +#include #include "iomfb_v14_7_board.h" +#include "iomfb_v14_7_swap.h" #define T6030_DCP_IMAGE "DDF38191-93B3-324A-BC8F-643006F5AC82" #define J613_DCP_IMAGE "90F849E1-B422-367E-B389-50246F8DEC47" @@ -179,12 +183,83 @@ static void v14_board_unchanged(struct kunit *test) KUNIT_EXPECT_TRUE(test, known); } +/* Only the internal panel of the J516C record stamps its swaps. */ +static void v14_board_swap_timestamps(struct kunit *test) +{ + const struct dcp_v14_board *j516c_b, *j514c_b, *t6030_b, *j613_b; + bool known; + + j516c_b = board_for("apple,t6031-dcp", j516c, &known); + j514c_b = board_for("apple,t6031-dcp", j514c, &known); + t6030_b = board_for("apple,t6030-dcp", j516s, &known); + j613_b = board_for("apple,t8122-dcp", j613, &known); + KUNIT_ASSERT_TRUE(test, j516c_b && j514c_b && t6030_b && j613_b); + + KUNIT_EXPECT_TRUE(test, j516c_b->swap_timestamps); + KUNIT_EXPECT_FALSE(test, j514c_b->swap_timestamps); + KUNIT_EXPECT_FALSE(test, t6030_b->swap_timestamps); + KUNIT_EXPECT_FALSE(test, j613_b->swap_timestamps); + + KUNIT_EXPECT_TRUE(test, dcp_v14_board_stamps_swaps(j516c_b, false)); + /* The J514C, the external role, the T6030 (J514S, J516S) and T8122 panels, no record. */ + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(j514c_b, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(j516c_b, true)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(t6030_b, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(j613_b, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(NULL, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(NULL, true)); +} + +/* + * A stamped swap holds 120 in timestamp slots 0, 1 and 6 and is otherwise the + * encoded record, with or without a surface; an unstamped one keeps every + * slot zero. + */ +static void v14_swap_timestamps_encoding(struct kunit *test) +{ + const size_t end = DCP_V14_SWAP_TIMESTAMP(DCP_V14_SWAP_TIMESTAMPS); + u8 surface[DCP_V14_SURFACE_SIZE]; + unsigned int i, pass; + u8 *plain, *stamped; + + plain = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + stamped = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, plain && stamped); + dcp_v14_encode_surface(surface, 3456 * 4, 3456, 2160, true); + + for (pass = 0; pass < 2; pass++) { + const u8 *s = pass ? surface : NULL; + + dcp_v14_encode_swap(plain, 0x2a, 0xff000000, s, 0x10000, pass ? 3456 : 0, + pass ? 2160 : 0, 74, 0); + dcp_v14_encode_swap(stamped, 0x2a, 0xff000000, s, 0x10000, pass ? 3456 : 0, + pass ? 2160 : 0, 74, 0); + dcp_v14_swap_timestamps(stamped); + + for (i = 0; i < DCP_V14_SWAP_TIMESTAMPS; i++) { + size_t at = DCP_V14_SWAP_TIMESTAMP(i); + u64 want = i == 0 || i == 1 || i == 6 ? 120 : 0; + + KUNIT_EXPECT_EQ_MSG(test, get_unaligned_le64(plain + at), 0, + "surface %u: unstamped slot %u", pass, i); + KUNIT_EXPECT_EQ_MSG(test, get_unaligned_le64(stamped + at), want, + "surface %u: slot %u", pass, i); + } + /* The rest of the record, from the word after slot 6, is unchanged. */ + KUNIT_EXPECT_EQ(test, + memcmp(plain + end, stamped + end, DCP_V14_SWAP_SIZE - end), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(stamped + DCP_V14_SWAP_ID), 0x2a); + } +} + static struct kunit_case v14_board_cases[] = { KUNIT_CASE(v14_board_j516c), KUNIT_CASE(v14_board_j514c), KUNIT_CASE(v14_board_t6031_other_boards), KUNIT_CASE(v14_board_j516c_image), KUNIT_CASE(v14_board_unchanged), + KUNIT_CASE(v14_board_swap_timestamps), + KUNIT_CASE(v14_swap_timestamps_encoding), {} }; From 94148052c842241c514d61cde07066cf0561901a Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:55 +0530 Subject: [PATCH 03/11] drm/asahi: g15: m3: account PB backing and re-initialization in the growth ledger The PB ledger only knows the posted count. For backing that is allocated after a reply instead of before it, the ledger also has to know how many blocks are backed, grant a request only from those, and start over at a re-initialization without forgetting the backing. Add `backed` to the ledger. `accept()` now returns an answer: - a grant, clipped to the backed blocks above the posted count; - a refusal at the maximum (the refusal bound is unchanged); - a wait when no backed block is left: the request keeps its counter and can be accepted again once more is backed. `reinit()` returns the posted count to the initial size and keeps the backing; Pools::init() uses it instead of a new ledger. The growth chunks that back_pb() allocates before an answer are now 128 blocks (16 MiB) instead of 32. answer_request() copies the backed count into the ledger after back_pb(), so no request waits yet. Host tests (pools/growth.rs, run by the asahi-g15 selftest): the clamps and the producer bound over the constructor's boundary values with the heap backed, growth to the maximum and re-initialization, and grants clipped to the backing, with waits that keep the counter. The ledger is shared by the T6030 and the T6031; this is untested on the T6030. The growth harness follows the chunk size and the ledger reset. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/asahi/g15/pools.rs | 18 +- drivers/gpu/drm/asahi/g15/pools/growth.rs | 195 +++++++++++++--------- tools/asahi/g15-pb-growth-tests.py | 54 +++--- 3 files changed, 161 insertions(+), 106 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index c635ed11c4a0c3..561de12d405ef9 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -7,7 +7,7 @@ //! //! - One 512 MiB PB heap at [`super::hw::client::PB_HEAP`] (4096 blocks of 128 KiB), backed on //! demand: the initial blocks when the GPU starts, then chunks of at most -//! [`growth::CHUNK_BLOCKS`] (4 MiB) as growth needs them ([`Firmware::back_pb`]), never freed +//! [`growth::CHUNK_BLOCKS`] (16 MiB) as growth needs them ([`Firmware::back_pb`]), never freed //! or shrunk. The backing ([`PbChunk`]) is mapped only into user VMs, at the fixed client GVA, //! each VM's before its prefixes are published ([`Firmware::map_pb`]); nothing names its pages //! through the host window or context 0 (the records hold client GVAs only). Initial @@ -377,10 +377,11 @@ impl Pools { true, )?; // Backing for the initial blocks; growth adds the rest. - let ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let mut ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); let mut chunks = KVec::with_capacity(growth::max_chunks(ledger.initial, ledger.max), GFP_KERNEL)?; chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?; + ledger.backed = ledger.initial; Ok(Self { dev: dev.as_ref().as_ref().into(), drm: dev.into(), @@ -451,8 +452,9 @@ impl Pools { // PB: 16384 pages of 0x8000 in 4096 blocks of 0x20000; with growth the first initial // blocks of them, at most 4095 posted, and page limits at the firmware's growth maximum. - // In every profile the block list declares its real 4096-record capacity. - self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + // The backing is kept. In every profile the block list declares its real 4096-record + // capacity. + self.ledger.reinit(); self.exhausted_logged = false; if tuning::PB_GROW { // C1 names the initial pages again. @@ -766,7 +768,13 @@ impl Firmware { // made so far stays for the next request. let end = self.pools.ledger.posted + self.pools.ledger.next_count(); self.back_pb(end, ctx)?; - let (first, n) = self.pools.ledger.accept(request); + self.pools.ledger.backed = self.pools.backed(); + let (first, n) = match self.pools.ledger.accept(request) { + growth::Answer::Grant { first, count } => (first, count), + growth::Answer::Refuse => (self.pools.ledger.posted, 0), + // Backed above: no request waits here. + growth::Answer::Wait => return Err(ENOSPC), + }; let heap = client::PB_HEAP; for b in first..first + n { self.pools.pb_blocks.w64( diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 20b298e641459d..0ce3696ebac556 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -12,6 +12,12 @@ //! its recovery path then apply, without a guarantee that they end the stall. A request from //! before a halt (epoch) or not matching a published render prefix is left unanswered. //! +//! The heap's VA range covers [`MAX_BLOCKS`] blocks; its RAM comes in chunks from the heap +//! base, which the ledger counts as `backed`. A request is granted from the backed blocks above +//! `posted`; a request that finds no backed block waits ([`Answer::Wait`]) and keeps its +//! counter. A re-initialization ([`Ledger::reinit`]) returns the posted count to the initial +//! size and keeps the backing. +//! //! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. use crate::g15::fw::buffer::{grow_request, BLOCK_LIST_RECORDS, GROW_STEP, MAX_BLOCKS, MIN_BLOCKS}; @@ -26,6 +32,8 @@ pub(crate) struct Ledger { pub(crate) max: u32, /// Blocks posted to the block list, cumulative (the block_ctl+4 producer is this modulo K). pub(crate) posted: u32, + /// Blocks with RAM behind them, from the heap base. + pub(crate) backed: u32, /// The next expected request counter. pub(crate) next_request: u32, /// A refusal at the maximum was logged since the last initialization. @@ -41,9 +49,20 @@ pub(crate) struct Ledger { /// Refusals answered at the maximum before requests are left to the deadline. pub(crate) const MAX_REFUSALS: u32 = 2; -/// Blocks of each PB backing chunk after the initial one (4 MiB): growth backs at most this -/// much at a time while the firmware waits for its answer. -pub(crate) const CHUNK_BLOCKS: u32 = 32; +/// Blocks of each PB backing chunk (16 MiB). +pub(crate) const CHUNK_BLOCKS: u32 = 128; + +/// The answer to an admitted growth request. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Answer { + /// Post blocks `[first, first + count)` (`count` at most [`GROW_STEP`]) and reply grown. + Grant { first: u32, count: u32 }, + /// Reply refused: the maximum is posted. + Refuse, + /// No backed block above the posted count: leave the request queued until a chunk is + /// backed. + Wait, +} /// The backing chunk that follows `backed` blocks of a ledger with maximum `max`: its first block /// and size, at most [`CHUNK_BLOCKS`] and never past `max`; none from `max` on. @@ -85,17 +104,32 @@ impl Ledger { initial, max, posted: initial, + backed: 0, next_request: 0, refusal_logged: false, refusals: 0, } } + /// A re-initialization: the initial blocks are posted again, the request counter and the + /// refusals start over, the backing stays. + pub(crate) const fn reinit(&mut self) { + self.posted = self.initial; + self.next_request = 0; + self.refusal_logged = false; + self.refusals = 0; + } + /// The live block_ctl+4 producer. pub(crate) const fn producer(&self) -> u32 { self.posted % BLOCK_LIST_RECORDS } + /// Backed blocks above the posted count. + pub(crate) const fn spare(&self) -> u32 { + self.backed.saturating_sub(self.posted) + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -114,21 +148,30 @@ impl Ledger { && request_epoch(request) == epoch } - /// The blocks the next accepted request posts: at most [`GROW_STEP`], none at the maximum. + /// The blocks the next accepted request posts with the backing it needs: at most + /// [`GROW_STEP`], none at the maximum. pub(crate) fn next_count(&self) -> u32 { GROW_STEP.min(self.max - self.posted) } - /// Accepts an admitted request: the next expected counter follows it, and the returned - /// blocks `[first, first + count)` are to be written to the block list, `count` 0 meaning a - /// refusal at the maximum. - pub(crate) fn accept(&mut self, request: &[u32; 14]) -> (u32, u32) { + /// Accepts an admitted request. Granted (at most [`GROW_STEP`] blocks, and no more than are + /// backed) or refused at the maximum, the next expected counter follows it. A request that + /// finds no backed block waits: its counter is not consumed, and it is accepted again once a + /// chunk is backed. + pub(crate) fn accept(&mut self, request: &[u32; 14]) -> Answer { + let room = self.max - self.posted; + if room != 0 && self.spare() == 0 { + return Answer::Wait; + } self.next_request = request[grow_request::COUNTER].wrapping_add(1); - let count = self.next_count(); - if count == 0 { + if room == 0 { self.refusals += 1; + return Answer::Refuse; + } + Answer::Grant { + first: self.posted, + count: GROW_STEP.min(room).min(self.spare()), } - (self.posted, count) } /// Records `count` blocks as posted after their records were written and returns the new @@ -185,12 +228,19 @@ mod tests { q } + /// A ledger with the heap backed whole. + fn backed(grow: bool, blocks: u32, max_blocks: u32) -> Ledger { + let mut l = Ledger::new(grow, blocks, max_blocks); + l.backed = MAX_BLOCKS; + l + } + #[test] fn clamps() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); assert_eq!( - (l.initial, l.max, l.posted, l.next_request), - (1024, 2048, 1024, 0) + (l.initial, l.max, l.posted, l.backed, l.next_request), + (1024, 2048, 1024, 4096, 0) ); assert_eq!(l.producer(), 1024); assert_eq!(Ledger::new(true, 10, 20).initial, 64); @@ -198,6 +248,7 @@ mod tests { assert_eq!(Ledger::new(true, 5000, 100).initial, 4095); assert_eq!(Ledger::new(true, 512, 1024).max, 1024); assert_eq!(Ledger::new(true, 512, 100).max, 512); + assert_eq!(Ledger::new(true, 512, 100).backed, 0); let fixed = Ledger::new(false, 512, 1024); assert_eq!((fixed.initial, fixed.max, fixed.posted), (4095, 4095, 4095)); // The values the driver passes (tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS) are kept. @@ -235,18 +286,19 @@ mod tests { for grow in [true, false] { for blocks in values { for max_blocks in values { - let mut l = Ledger::new(grow, blocks, max_blocks); + let mut l = backed(grow, blocks, max_blocks); assert!(MIN_BLOCKS <= l.initial && l.initial <= l.max && l.max <= MAX_POSTED); assert_eq!(l.posted, l.initial); assert!((1..=MAX_POSTED).contains(&l.producer())); // Every producer of a growth to the maximum stays in 1..=4095. let mut counter = 0; loop { - let (first, n) = l.accept(&request(5, counter, 1)); + let (first, n) = match l.accept(&request(5, counter, 1)) { + Answer::Grant { first, count } => (first, count), + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert_eq!(first, l.posted); - if n == 0 { - break; - } let producer = l.post(n); assert!((1..=MAX_POSTED).contains(&producer)); assert_eq!(producer, l.posted); @@ -261,7 +313,7 @@ mod tests { #[test] fn admission() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); let q = request(5, 0, 3); assert!(l.admits(&q, true, true, 3)); assert!(!l.admits(&q, false, true, 3)); @@ -283,17 +335,20 @@ mod tests { #[test] fn growth_to_the_maximum() { // 512 initial and 1024 maximum blocks. - let mut l = Ledger::new(true, 512, 1024); + let mut l = backed(true, 512, 1024); let mut grown = 0; let mut counter = 0; loop { let q = request(5, counter, 1); assert!(l.admits(&q, true, true, 1)); - let (first, n) = l.accept(&q); - assert_eq!(first, 512 + grown); - if n == 0 { - break; - } + let n = match l.accept(&q) { + Answer::Grant { first, count } => { + assert_eq!(first, 512 + grown); + count + } + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert!(n <= 10); assert_eq!(l.post(n), (512 + grown + n) % 4096); grown += n; @@ -310,69 +365,57 @@ mod tests { assert_eq!(l.refusals, 1); assert!(!l.exhausted()); let q = request(5, counter + 3, 1); - assert_eq!(l.accept(&q), (1024, 0)); + assert_eq!(l.accept(&q), Answer::Refuse); assert!(l.exhausted()); assert!(!l.admits(&request(5, counter + 4, 1), true, true, 1)); - // A fresh ledger starts over. + // A fresh ledger starts over, and so does a re-initialized one (backing kept). assert!(!Ledger::new(true, 512, 1024).exhausted()); - } - - /// Backing for every ledger the constructor makes, the driver's included: the chunks from - /// the initial one are contiguous, at most [`CHUNK_BLOCKS`] each, end at the maximum and - /// number at most [`max_chunks`]; growth with the backing made first never posts a block - /// past it. - #[test] - fn backing_chunks() { - assert_eq!(max_chunks(1024, 4095), 1 + 96); - assert_eq!(max_chunks(MIN_BLOCKS, MAX_POSTED), 1 + 126); - assert_eq!(max_chunks(MAX_POSTED, MAX_POSTED), 1); - assert_eq!(next_chunk(1024, 4095), Some((1024, 32))); - assert_eq!(next_chunk(4064, 4095), Some((4064, 31))); - assert_eq!(next_chunk(4095, 4095), None); - assert_eq!(next_chunk(4096, 4095), None); - let values = [ - 0, 1, MIN_BLOCKS, 65, 1023, 1024, 1025, 3072, 4094, MAX_POSTED, 5000, - ]; - for grow in [true, false] { - for blocks in values { - for max_blocks in values { - let mut l = Ledger::new(grow, blocks, max_blocks); - let (mut backed, mut chunks) = (l.initial, 1); - let mut counter = 0; - loop { - let end = l.posted + l.next_count(); - while backed < end { - let (first, n) = next_chunk(backed, l.max).unwrap(); - assert_eq!(first, backed); - assert!(n >= 1 && n <= CHUNK_BLOCKS); - backed += n; - chunks += 1; - } - let (first, n) = l.accept(&request(5, counter, 1)); - assert!(first + n <= backed); - if n == 0 { - break; - } - l.post(n); - counter += 1; - } - assert_eq!((l.posted, backed), (l.max, l.max)); - assert!(chunks <= max_chunks(l.initial, l.max)); - assert_eq!(next_chunk(backed, l.max), None); - } - } - } + l.reinit(); + assert_eq!( + (l.posted, l.next_request, l.refusals, l.backed), + (512, 0, 0, 4096) + ); + assert!(!l.exhausted() && !l.refusal_logged); } #[test] fn producer_wraps_at_k() { - let mut l = Ledger::new(true, 2048, 2048); + let mut l = backed(true, 2048, 2048); assert_eq!(l.producer(), 2048); l.posted = 4095; assert_eq!(l.post(1), 0); assert_eq!(l.post(10), 10); } + #[test] + fn grants_only_backed_blocks() { + let mut l = Ledger::new(true, 64, 4095); + // Nothing backed above the initial posting: a request waits and keeps its counter. + l.backed = 64; + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + // A grant is clipped to the backed blocks; the next request waits again. + l.backed = 70; + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 6 + } + ); + l.post(6); + assert_eq!(l.next_request, 1); + assert_eq!(l.accept(&request(3, 1, 1)), Answer::Wait); + // At the maximum the refusal does not wait for backing. + l.posted = l.max; + l.backed = l.max; + assert_eq!(l.accept(&request(3, 1, 1)), Answer::Refuse); + // A re-initialization keeps the backing. + l.reinit(); + assert_eq!((l.posted, l.backed, l.next_request), (64, 4095, 0)); + } + #[test] fn logging() { let mut l = Ledger::new(true, 64, 64); diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 7cdedc9f0cc951..78cca50e19e109 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -27,7 +27,7 @@ exactly is not; - invalid requests (type, slot, stale counter, epoch, no live render) get no reply, and requests rejected so allocate nothing; -- the backing: the initial blocks only when the GPU starts; chunks of at most 4 MiB, allocated +- the backing: the initial blocks only when the GPU starts; chunks of at most 16 MiB, allocated and held by the running VM before any record or producer names their blocks, never past the maximum, at most growth::max_chunks(); a failed allocation, alias map or append posts and answers nothing, leaves the request counter and the VM's count alone and drops what is not @@ -96,12 +96,12 @@ }''' # Pools::new's PB backing, run verbatim by the model. -POOLS_NEW_BACKING = (' // Backing for the initial blocks; growth adds the rest.\n', ' chunks.push(') +POOLS_NEW_BACKING = (' // Backing for the initial blocks; growth adds the rest.\n', ' ledger.backed = ') # The compile-time proof that a PB alias drop is a plain unmap. PB_LEAF_ASSERT = 'static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent());' # The two statements of Pools::init() that start a new ledger. -LEDGER_RESET = ['self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS);', +LEDGER_RESET = ['self.ledger.reinit();', 'self.exhausted_logged = false;'] MUTATIONS = { @@ -138,12 +138,14 @@ 'let nb = clamp(blocks, MIN_BLOCKS, MAX_BLOCKS);\n' ' (nb, clamp(max_blocks, nb, MAX_BLOCKS))\n' ' } else {\n (MAX_BLOCKS, MAX_BLOCKS)'), - # Post blocks without backing them first. - 'grant-without-backing': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', - ' let (first, n) = self.pools.ledger.accept(request);'), + # Post blocks without backing them first (the ledger told the whole heap is backed). + 'grant-without-backing': (POOLS, ' self.back_pb(end, ctx)?;\n self.pools.ledger.backed = self.pools.backed();\n', + ' self.pools.ledger.backed = buffer::MAX_BLOCKS;\n'), # Back the blocks only after the request is accepted. - 'back-after-accept': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', - ' let (first, n) = self.pools.ledger.accept(request);\n self.back_pb(end, ctx)?;'), + 'back-after-accept': [(POOLS, ' self.back_pb(end, ctx)?;\n self.pools.ledger.backed = self.pools.backed();\n', + ' self.pools.ledger.backed = buffer::MAX_BLOCKS;\n'), + (POOLS, ' growth::Answer::Wait => return Err(ENOSPC),\n };\n', + ' growth::Answer::Wait => return Err(ENOSPC),\n };\n self.back_pb(end, ctx)?;\n')], # Count a catch-up alias before the VM holds it. 'catch-up-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n mapped += 1;\n' ' vm.pb_chunks.store(mapped, Ordering::Relaxed);', @@ -172,8 +174,8 @@ # Back the whole heap when the GPU starts. 'initial-backs-whole-heap': (POOLS, 'chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?;', 'chunks.push(PbChunk::new(dev, 0, buffer::MAX_BLOCKS)?, GFP_KERNEL)?;'), - # Growth chunks of 8 MiB. - 'chunks-above-4mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 32;', 'pub(crate) const CHUNK_BLOCKS: u32 = 64;'), + # Growth chunks of 32 MiB. + 'chunks-above-16mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 128;', 'pub(crate) const CHUNK_BLOCKS: u32 = 256;'), # A firmware-visible cached PB leaf (its drop would ask the firmware to flush). 'pb-leaf-fw-visible': (HW, 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c83;', 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c03;'), @@ -202,7 +204,7 @@ 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), 'no-same-vm-checks': ('grant_refuses_flights_of_two_vms', 'grant with flights of two VMs'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), - 'chunks-above-4mib': ('growth_backs_before_posting', 'a growth chunk above 4 MiB'), + 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), 'pb-leaf-fw-visible': ('pb_leaf_is_gpu_only', 'PB leaf not GPU-only'), } assert set(INTENDED) == set(MUTATIONS) @@ -978,7 +980,9 @@ def consts(text, names): assert!(log.iter().all(|e| matches!(e, Ev::Log("warn", _))), "{log:?}"); assert_eq!(log.len(), 4); assert_eq!(f.pools.stats.bad, 4); - assert_eq!(f.pools.ledger, Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS)); + let mut fresh = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + fresh.backed = fresh.initial; + assert_eq!(f.pools.ledger, fresh); // No render published: not live. f.renders[0] = 0; f.events.grow.0.push_back(good); @@ -1072,7 +1076,7 @@ def consts(text, names): assert_eq!(log, vec![Ev::Alloc(1, initial as usize * BLOCK_BYTES as usize)], "startup allocation"); assert_eq!((f.pools.backed(), f.pools.chunk_count()), (initial, 1)); assert_eq!(f.pools.chunk_capacity(), max_chunks(initial, max)); - assert_eq!(max_chunks(initial, max), 97); + assert_eq!(max_chunks(initial, max), 25); // The first prefix of a VM maps the backing, nothing else. let vm = VmState::new(7); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); @@ -1107,14 +1111,14 @@ def consts(text, names): let log = take(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); assert_posted_backed(&log, 5, initial as u64); - // Chunks of at most 32 blocks, allocated only as needed: up to the maximum, no further. + // Chunks of at most 128 blocks, allocated only as needed: up to the maximum, no further. let allocs: Vec = log .iter() .filter_map(|e| if let Ev::Alloc(_, b) = e { Some(*b) } else { None }) .collect(); assert_eq!(allocs.len() + 1, max_chunks(initial, max)); - assert!(allocs.iter().all(|b| *b <= 4 << 20), "a growth chunk above 4 MiB"); - assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 4 << 20); + assert!(allocs.iter().all(|b| *b <= 16 << 20), "a growth chunk above 16 MiB"); + assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 16 << 20); assert_eq!(f.pools.backed(), max); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); assert_cursor(&f.engine.flights[0].job.vm, f.pools.chunk_count()); @@ -1185,7 +1189,7 @@ def consts(text, names): let a = f.engine.flights[0].job.vm.clone(); grow_to(&mut f, 1100); take(); - assert_eq!(f.pools.chunk_count(), 4); + assert_eq!(f.pools.chunk_count(), 2); // A's work retires; a new VM runs next: it maps every chunk first. f.engine.flights.clear(); let b = VmState::new(6); @@ -1193,13 +1197,13 @@ def consts(text, names): f.fw.ctx = 6; f.map_pb(&b).unwrap(); let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Append(6, _))), 4); + assert_eq!(count(&log, |e| matches!(e, Ev::Append(6, _))), 2); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 0); - assert_cursor(&b, 4); + assert_cursor(&b, 2); // B grows the backing; A, behind, catches up when it runs again. grow_to(&mut f, 1200); let log = take(); - assert_posted_backed(&log, 6, 1120); + assert_posted_backed(&log, 6, 1152); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 0); f.engine.flights.clear(); f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); @@ -1234,7 +1238,7 @@ def consts(text, names): #[test] fn partial_catch_up_counts_only_held_aliases() { let mut f = Firmware::new(); - grow_to(&mut f, 1100); + grow_to(&mut f, 1450); f.engine.flights.clear(); let b = VmState::new(6); f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); @@ -1244,17 +1248,17 @@ def consts(text, names): crate::MAP_OK_BEFORE_FAIL.with(|c| c.set(2)); FAIL_MAP.with(|c| c.set(1)); assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_cursor(&b, 4); + assert_cursor(&b, 5); assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2); // The third alias is made but cannot be held: dropped, still two. FAIL_APPEND.with(|c| c.set(1)); assert_eq!(f.map_pb(&b), Err(ENOMEM)); assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2, "count after a failed append"); assert!(take().iter().any(|e| matches!(e, Ev::Unmap(6, _)))); - assert_cursor(&b, 4); + assert_cursor(&b, 5); f.map_pb(&b).unwrap(); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 4); - assert_cursor(&b, 4); + assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 5); + assert_cursor(&b, 5); } /// The ledger reset of Pools::init(), not a full recovery. From 7eae5b05328ca1d86ac1e770a5170829a12f4ec6 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:56 +0530 Subject: [PATCH 04/11] drm/asahi: g15: m3: back the PB in 16 MiB chunks after the reply, with a reserve back_pb() allocates and maps the backing of a growth request before the answer, while the firmware waits for it. The allocation has no time bound (shmem pages with __GFP_RETRY_MAYFAIL), and a failure leaves the request unanswered until the host deadline. Keep one chunk of backed blocks in reserve above the posted count and back the next chunk after the replies of a poll pass: - the heap is backed in chunks of 128 blocks (16 MiB), each a gpu_backing() object at its fixed client GVA (chunk_gva()); - when the GPU starts, the chunks of the initial posting and one reserve chunk are backed (9 with the initial 1024 blocks); - answer_request() grants only from backed blocks; while fewer than 128 remain above the posted count, refill_backing() backs chunks at the end of answer_growth(), after every reply of the pass; - with the reserve dry (an allocation failed), a request waits: it goes back to the head of the queue with its counter and is answered by a later pass once a chunk is backed; - a pending re-initialization backs nothing while no render is published; a render published before it (the flights behind a fragment kill) is still backed, or its requests would wait until the deadline. A compute-only flight gets none: it cannot ask for growth. Mapping stays per VM and before the VM can reach a block. A VM maps every chunk it lacks at the start of its prefix's publication (map_pb(), with its one-VM check), and a chunk backed by a refill is mapped into the VM of the flights before the ledger counts it. If that mapping fails, the chunk is dropped again and the next pass retries. The aliases are held in the VM's own heap list (VmState::map_heap()), which keeps the object of each alias. A chunk whose object changed is unmapped and mapped again, and aliases past the backing are dropped. back_pb(), running_vm() and the per-VM count are removed: no grant can name a block that the running VM has not mapped. This applies to the T6030 and the T6031, which share this code; it is untested on the T6030. The debugfs pb: line appends waits and refills. The growth harness now runs answer_growth(), answer_request(), map_pb(), refill_backing() and the VM heap-mapping bodies. Its new cases: - the start backing; - allocation only after the pass's replies; - a dry reserve that makes a request wait and keeps its order; - a failed refill map that drops the chunk; - a pending re-initialization with no flight, with a compute-only flight, and with a render published before it; - catch-up at publication, including a partial one. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/asahi/g15/pools.rs | 345 +++++++------ drivers/gpu/drm/asahi/g15/pools/growth.rs | 163 ++++-- drivers/gpu/drm/asahi/g15/vm.rs | 91 +++- tools/asahi/g15-pb-growth-tests.py | 603 ++++++++++++---------- 4 files changed, 739 insertions(+), 463 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index 561de12d405ef9..15204f60916138 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -6,16 +6,21 @@ //! (host-tested). //! //! - One 512 MiB PB heap at [`super::hw::client::PB_HEAP`] (4096 blocks of 128 KiB), backed on -//! demand: the initial blocks when the GPU starts, then chunks of at most -//! [`growth::CHUNK_BLOCKS`] (16 MiB) as growth needs them ([`Firmware::back_pb`]), never freed -//! or shrunk. The backing ([`PbChunk`]) is mapped only into user VMs, at the fixed client GVA, -//! each VM's before its prefixes are published ([`Firmware::map_pb`]); nothing names its pages -//! through the host window or context 0 (the records hold client GVAs only). Initial +//! demand in chunks of [`growth::CHUNK_BLOCKS`] blocks (16 MiB) from the heap base: when the +//! GPU starts, the chunks of the initial posting and one reserve chunk +//! ([`growth::Ledger::chunks_for`]). The backing ([`PbChunk`]) is mapped only into user VMs, at +//! the fixed client GVA: each VM's before its prefixes are published ([`Firmware::map_pb`]), +//! and a chunk backed later also into the VM of the flights; nothing names its pages through +//! the host window or context 0 (the records hold client GVAs only). Initial //! [`super::hw::tuning::PB_BLOCKS`] (1024), at most [`super::hw::tuning::PB_MAX_BLOCKS`] //! (4095) posted, so the producer never returns to 0; the firmware is given //! [`Pools::soft_max`] (default 3072) as its growth maximum. Truthful K = 4096 in every //! profile (PB reconstruction, also in recovery, synchronizes 8 * K bytes of the block list: //! K = 0x26cd overran the 0x8000-byte list and crashed the second recovery). +//! - Backing: a request is granted from the backed blocks above the posted count; after the +//! replies of a pass, while fewer than a reserve chunk remain, [`Firmware::refill_backing`] +//! backs the next chunk, so a reply never waits for the allocator. Only a reserve that ran dry +//! makes a request wait: it stays queued, the pass backs a chunk and the next pass answers it. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -41,7 +46,6 @@ mod growth; use core::fmt; -use core::sync::atomic::Ordering; use kernel::{ device, prelude::*, @@ -58,7 +62,7 @@ use super::Firmware; use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{gem, mmu}; -pub(crate) use growth::Ledger; +pub(crate) use growth::{Answer, Ledger}; static_assert!(buffer::MAX_BLOCKS == sizes::PB_BLOCKS); static_assert!(buffer::MAX_PAGES == sizes::PB_PAGES); @@ -75,47 +79,36 @@ static_assert!(tuning::PB_BLOCKS <= tuning::PB_MAX_BLOCKS); // The PB alias is GPU-only (not cached and firmware-visible): dropping one is a plain unmap, // with no firmware cache flush (`KernelMapping::drop`). static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent()); +static_assert!(growth::MAX_CHUNKS * growth::CHUNK_BLOCKS == buffer::MAX_BLOCKS); -/// Physical backing of PB blocks `[first, first + blocks)`: a zeroed GEM kernel object, pinned -/// and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at the blocks' fixed -/// client GVA ([`Firmware::map_pb`]). Retained until reboot, like every firmware object; each -/// alias also holds the object. +/// Bytes of one backing chunk. +pub(crate) const CHUNK_BYTES: usize = growth::CHUNK_BLOCKS as usize * buffer::BLOCK_BYTES as usize; + +/// Client GVA of backing chunk `i`. +pub(crate) const fn chunk_gva(i: usize) -> u64 { + client::PB_HEAP + (i * CHUNK_BYTES) as u64 +} + +/// Physical backing of one chunk of PB blocks: a zeroed GEM kernel object of [`CHUNK_BYTES`], +/// pinned and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at its fixed +/// client GVA ([`chunk_gva`], [`VmState::map_heap`]). Each alias also holds the object. struct PbChunk { - first: u32, - blocks: u32, _pages: HeldPages, backing: gem::ObjectRef, } impl PbChunk { - fn new(dev: &AsahiDevice, first: u32, blocks: u32) -> Result { - if blocks == 0 || first.checked_add(blocks).ok_or(EINVAL)? > buffer::MAX_BLOCKS { - return Err(EINVAL); - } - let (backing, pages) = - object::gpu_backing(dev, blocks as usize * buffer::BLOCK_BYTES as usize)?; + fn new(dev: &AsahiDevice) -> Result { + let (backing, pages) = object::gpu_backing(dev, CHUNK_BYTES)?; Ok(Self { - first, - blocks, _pages: pages, backing, }) } - /// The block after the chunk's last. - fn end(&self) -> u32 { - self.first + self.blocks - } - - /// Maps the whole chunk, exactly its object's size, at its fixed client GVA in `vm`. - fn map(&self, vm: &mmu::Vm) -> Result { - vm.map_at( - client::PB_HEAP + self.first as u64 * buffer::BLOCK_BYTES, - self.blocks as usize * buffer::BLOCK_BYTES as usize, - &self.backing.gem, - mmu::Prot::from_pte(pte::LEAF_CACHED), - false, - ) + /// The GEM object, for the VM mappings. + fn gem(&self) -> ARef { + self.backing.gem.clone() } } @@ -161,6 +154,10 @@ pub(crate) struct PbStats { pub(crate) grown: u64, pub(crate) refused: u64, pub(crate) bad: u64, + /// Requests that waited a pass for backing. + pub(crate) waits: u64, + /// Chunks backed after the GPU started. + pub(crate) refills: u64, } /// Render allocator and PB state: the allocators, the PB ledger and the spare set. @@ -179,9 +176,8 @@ pub(crate) struct Pools { spare_ready: bool, pb_pages: Arc, pb_blocks: Arc, - /// The PB backing, contiguous from block 0, in block order: the initial blocks, then the - /// growth chunks. Its capacity, reserved up front, is the most the ledger can need - /// ([`growth::max_chunks`]). + /// The PB backing chunks, from the heap base (chunk `i` at [`chunk_gva`]`(i)`). Its + /// capacity, reserved up front, is the whole heap's [`growth::MAX_CHUNKS`]. chunks: KVec, /// The PB growth ledger since the last (re)initialization. pub(crate) ledger: Ledger, @@ -376,12 +372,20 @@ impl Pools { 0, true, )?; - // Backing for the initial blocks; growth adds the rest. + // Backing for the initial blocks and the reserve; growth adds the rest. let mut ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); - let mut chunks = - KVec::with_capacity(growth::max_chunks(ledger.initial, ledger.max), GFP_KERNEL)?; - chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?; - ledger.backed = ledger.initial; + let mut chunks = KVec::with_capacity(growth::MAX_CHUNKS as usize, GFP_KERNEL)?; + for _ in 0..Ledger::chunks_for(ledger.initial) { + chunks.push(PbChunk::new(dev)?, GFP_KERNEL)?; + ledger.back_chunk(); + } + dev_info!( + dev.as_ref(), + "G15: PB heap: {} blocks posted at start, {} chunks of {} MiB backed\n", + ledger.initial, + chunks.len(), + CHUNK_BYTES >> 20 + ); Ok(Self { dev: dev.as_ref().as_ref().into(), drm: dev.into(), @@ -472,7 +476,7 @@ impl Pools { } let heap = client::PB_HEAP; let initial = self.ledger.initial; - if initial > self.backed() { + if initial > self.ledger.backed { return Err(EINVAL); } write_records( @@ -526,133 +530,130 @@ impl Pools { super::params::pb_max_blocks().clamp(self.ledger.initial, self.ledger.max) } - /// The blocks with backing, from block 0. - pub(crate) fn backed(&self) -> u32 { - self.chunks.last().map_or(0, PbChunk::end) + /// Maps the backed chunks `vm` does not hold yet ([`VmState::map_heap`]). + fn map_heap(&self, vm: &VmState) -> Result { + vm.map_heap(self.chunks.iter().map(PbChunk::gem)) + } + + /// Backs one more chunk; the caller maps it where needed before the ledger posts from it. + fn back_chunk(&mut self) -> Result> { + if self.chunks.len() >= growth::MAX_CHUNKS as usize { + return Err(ENOSPC); + } + let chunk = PbChunk::new(&self.drm)?; + let gem = chunk.gem(); + self.chunks.push(chunk, GFP_KERNEL)?; + Ok(gem) } /// The debugfs line `pb: blocks initial/posted/firmware-max/max backed blocks/chunks - /// requests grown refused bad`. + /// requests grown refused bad waits refills` (new keys are appended). pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { writeln!( f, - "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {}", + "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {} waits {} refills {}", self.ledger.initial, self.ledger.posted, self.soft_max(), self.ledger.max, - self.backed(), + self.ledger.backed, self.chunks.len(), self.stats.requests, self.stats.grown, self.stats.refused, - self.stats.bad + self.stats.bad, + self.stats.waits, + self.stats.refills ) } } impl Firmware { - /// Maps the PB backing into `vm` up to its last chunk, before anything of a prefix of the - /// VM is published (the start of `publish_prefix_into`). Render work runs in one VM at a - /// time (`Engine::publish`), so the VM about to run is the only one through which the GPU - /// can reach a posted block next; a VM created or idle while the backing grew catches up - /// here, also a closed VM whose accepted jobs still run. + /// Maps the PB backing into `vm`, before anything of a prefix of the VM is published (the + /// start of `publish_prefix_into`). Render work runs in one VM at a time + /// (`Engine::publish`), so the VM about to run is the only one through which the GPU can + /// reach a posted block next; a VM created or idle while the backing grew catches up here, + /// also a closed VM whose accepted jobs still run. A chunk is mapped only once + /// ([`VmState::map_heap`]); one that fails leaves the chunks mapped before it, and the + /// prefix is not published. The leaf is GPU-only, so dropping an alias is a plain unmap. /// - /// Each chunk's alias joins the VM's driver mappings, held until the VM is torn down, and - /// the VM's count moves past the chunk only then. When that fails, the alias just made is - /// dropped again: nothing of the VM runs before its count covers the chunk, and the leaf is - /// GPU-only, so the drop is a plain unmap. The chunks mapped before stay counted. - /// - /// The one-VM rule is checked first: a flight of another VM, or a count past the chunks, - /// fails with EPROTO before anything is mapped. + /// The one-VM rule is checked first: a flight of another VM fails with EPROTO before + /// anything is mapped. pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result { - let mut mapped = vm.pb_chunks.load(Ordering::Relaxed); - if mapped > self.pools.chunks.len() - || self - .engine - .flights - .iter() - .any(|f| !core::ptr::eq(&*f.job.vm, vm)) - { + let others = self + .engine + .flights + .iter() + .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) + .count(); + if others != 0 { dev_err!( self.dev(), - "PB backing: ctx {} maps {} of {} chunks with {} flights of another VM\n", + "PB backing: ctx {} maps {} chunks with {} flights of another VM\n", vm.slot, - mapped, self.pools.chunks.len(), - self.engine - .flights - .iter() - .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) - .count() + others ); return Err(super::boot::eproto()); } - while let Some(chunk) = self.pools.chunks.get(mapped) { - let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; - aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; - vm.vm.append_driver_mappings(aliases)?; - mapped += 1; - vm.pb_chunks.store(mapped, Ordering::Relaxed); - } - Ok(()) + self.pools.map_heap(vm) } - /// The VM whose render work runs: every flight's (`Engine::publish` puts a flight behind - /// another only for the same VM), and it must be the asking context's. EPROTO otherwise. - fn running_vm(&self, ctx: u32) -> Result> { - let flights = &self.engine.flights; - let vm = flights - .first() - .ok_or_else(super::boot::eproto)? - .job - .vm - .clone(); - if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) { - dev_err!( - self.dev(), - "PB growth: ctx {} is not the one VM in flight (ctx {}, {} flights)\n", - ctx, - vm.slot, - flights.len() - ); - return Err(super::boot::eproto()); + /// Backs chunks until the reserve is whole, mapping each into the VMs of the flights (by + /// the one-VM rule, the VM that runs) before the ledger may post from it; any other VM + /// maps it at its next publication ([`Self::map_pb`]). A failure is logged and leaves the + /// reserve short: requests then wait, and the next pass tries again. + pub(crate) fn refill_backing(&mut self) { + // A pending re-initialization is about to drop the posted count: nothing to back for, + // unless a render published before it still runs (the flights behind a fragment kill + // keep asking for growth until they retire). + let render = (0..self.engine.flights.len()).any(|i| self.receipts(i).renders != 0); + if self.recovery.reinit && !render { + return; } - Ok(vm) - } - - /// Backs the PB up to block `end` in the VM that runs, the asking context `ctx`, before an - /// answer posts those blocks (nothing new when they are backed already): chunks of at most - /// [`growth::CHUNK_BLOCKS`] blocks are allocated ([`object::gpu_backing`]), mapped into that - /// VM and kept, one at a time. The VM first catches up ([`Self::map_pb`], which also checks - /// the one-VM rule), so its count equals the chunks. A chunk is kept - /// only once its alias in the VM is held, and the VM's count moves with it; when the alias - /// cannot be held, the alias and the chunk are dropped, neither ever reachable (the leaf is - /// GPU-only: a plain unmap). An error leaves the chunks backed so far for the next request, - /// and nothing is posted. The allocation sleeps (shmem pages with `__GFP_RETRY_MAYFAIL`) - /// while the firmware waits for the answer; how long is not bounded here. - fn back_pb(&mut self, end: u32, ctx: u32) -> Result { - let vm = self.running_vm(ctx)?; - self.map_pb(&vm)?; - while self.pools.backed() < end { - let (first, blocks) = - growth::next_chunk(self.pools.backed(), self.pools.ledger.max).ok_or(ENOSPC)?; - if self.pools.chunks.len() == self.pools.chunks.capacity() { - return Err(ENOSPC); + while self.pools.ledger.refill_due() { + let result = (|| -> Result { + let gem = self.pools.back_chunk()?; + let i = self.pools.chunks.len() - 1; + for flight in self.engine.flights.iter() { + flight.job.vm.map_chunk(i, gem.clone())?; + } + Ok(()) + })(); + match result { + Ok(()) => { + self.pools.ledger.back_chunk(); + self.pools.stats.refills += 1; + dev_dbg!( + self.dev(), + "PB heap: chunk {} backed ({} of {} blocks posted)\n", + self.pools.chunks.len() - 1, + self.pools.ledger.posted, + self.pools.ledger.backed + ); + } + Err(e) => { + // A chunk that could not be mapped into a flight's VM is not counted: drop + // it (and its aliases in the VMs) again. + let backed = (self.pools.ledger.backed / growth::CHUNK_BLOCKS) as usize; + if self.pools.chunks.len() > backed { + for flight in self.engine.flights.iter() { + flight.job.vm.unmap_heap_above(backed); + } + self.pools.chunks.truncate(backed); + } + dev_warn!( + self.dev(), + "PB heap: backing chunk {} failed: {} ({} of {} blocks posted)\n", + backed, + e.to_errno(), + self.pools.ledger.posted, + self.pools.ledger.backed + ); + return; + } } - let chunk = PbChunk::new(&self.pools.drm, first, blocks)?; - let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; - aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; - vm.vm.append_driver_mappings(aliases)?; - // Within the capacity checked above. - self.pools - .chunks - .push_within_capacity(chunk) - .map_err(|_| EIO)?; - vm.pb_chunks - .store(self.pools.chunks.len(), Ordering::Relaxed); } - Ok(()) } /// (Re)initializes the render UMA managers, PB lists and PB manager. `rebuild`: the priming @@ -685,9 +686,12 @@ impl Firmware { /// DEVCTL 7 reports success; at the ledger's maximum the reply is the refusal (+4 = 0), /// which gives the firmware no new pages, and after two refusals requests are left /// unanswered. The firmware imports the blocks itself: P is never rewritten. Only the - /// requests queued before the pass are taken. A failed reply is logged; a request that - /// cannot be processed at all (a firmware object access failed) is logged and left - /// unanswered like an invalid one. Never fails: the poll pass continues. + /// requests queued before the pass are taken. A request with no backed block left waits: + /// it goes back to the head of the queue for the next pass, and this pass ends its answers + /// there. After the answers the reserve is refilled ([`Self::refill_backing`]). A failed + /// reply is logged; a request that cannot be processed at all (a firmware object access + /// failed) is logged and left unanswered like an invalid one. Never fails: the poll pass + /// continues. pub(crate) fn answer_growth(&mut self) -> Result { // Only the requests queued before this pass: a reply's wait drains the rings and may // queue the next request, which the next pass answers (the device lock is released in @@ -697,23 +701,36 @@ impl Firmware { let Ok(request) = self.events.grow.remove(0) else { break; }; - self.pools.stats.requests += 1; - if let Err(e) = self.answer_request(&request) { - self.pools.stats.bad += 1; - dev_err!( - self.dev(), - "PB growth request {} ctx {} not answered: {}\n", - request[buffer::grow_request::COUNTER], - request[buffer::grow_request::CONTEXT], - e.to_errno() - ); + match self.answer_request(&request) { + Ok(true) => self.pools.stats.requests += 1, + Ok(false) => { + // Waits for backing: back at the head, counted once answered. + self.pools.stats.waits += 1; + // The queue has room: this request was just taken from it. + if self.events.grow.insert_within_capacity(0, request).is_err() { + self.pools.stats.bad += 1; + } + break; + } + Err(e) => { + self.pools.stats.requests += 1; + self.pools.stats.bad += 1; + dev_err!( + self.dev(), + "PB growth request {} ctx {} not answered: {}\n", + request[buffer::grow_request::COUNTER], + request[buffer::grow_request::CONTEXT], + e.to_errno() + ); + } } } + self.refill_backing(); Ok(()) } - /// Processes one growth request. - fn answer_request(&mut self, request: &[u32; 14]) -> Result { + /// Processes one growth request: `Ok(false)` when it waits for backing. + fn answer_request(&mut self, request: &[u32; 14]) -> Result { let ctl = self.init.control.clone().ok_or(ENODEV)?; let epoch = ctl.r32(control::HALT_COUNT)? as u64 | (ctl.r32(control::HALT_COUNT + 4)? as u64) << 32; @@ -739,7 +756,7 @@ impl Firmware { self.pools.ledger.max ); } - return Ok(()); + return Ok(true); } // A request from before a halt has no authority over the PB now. if !self @@ -760,20 +777,14 @@ impl Firmware { live as u32, tuning::PB_GROW as u32 ); - return Ok(()); + return Ok(true); } - // Every block the answer posts is backed in the VM that runs before anything of the - // answer is written; the one-VM rule is checked for every answer. If that fails, nothing - // is posted or answered (answer_growth logs it; the deadline applies), and the backing - // made so far stays for the next request. - let end = self.pools.ledger.posted + self.pools.ledger.next_count(); - self.back_pb(end, ctx)?; - self.pools.ledger.backed = self.pools.backed(); + // A grant posts only backed blocks, and every backed chunk is mapped in the VM of the + // flights (at its publication, or at the refill that backed it). let (first, n) = match self.pools.ledger.accept(request) { - growth::Answer::Grant { first, count } => (first, count), - growth::Answer::Refuse => (self.pools.ledger.posted, 0), - // Backed above: no request waits here. - growth::Answer::Wait => return Err(ENOSPC), + Answer::Grant { first, count } => (first, count), + Answer::Refuse => (self.pools.ledger.posted, 0), + Answer::Wait => return Ok(false), }; let heap = client::PB_HEAP; for b in first..first + n { @@ -839,6 +850,6 @@ impl Firmware { ret ); } - Ok(()) + Ok(true) } } diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 0ce3696ebac556..3ed143ee546ad5 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -12,11 +12,13 @@ //! its recovery path then apply, without a guarantee that they end the stall. A request from //! before a halt (epoch) or not matching a published render prefix is left unanswered. //! -//! The heap's VA range covers [`MAX_BLOCKS`] blocks; its RAM comes in chunks from the heap -//! base, which the ledger counts as `backed`. A request is granted from the backed blocks above -//! `posted`; a request that finds no backed block waits ([`Answer::Wait`]) and keeps its -//! counter. A re-initialization ([`Ledger::reinit`]) returns the posted count to the initial -//! size and keeps the backing. +//! The heap's VA range covers [`MAX_BLOCKS`] blocks; its RAM comes in chunks of +//! [`CHUNK_BLOCKS`] blocks from the heap base, which the ledger counts as `backed`. A request is +//! granted from the backed blocks above `posted`. A chunk is due ([`Ledger::refill_due`]) +//! whenever fewer than [`RESERVE_BLOCKS`] backed blocks remain above the posted count, so that +//! it can be backed after a reply; a request that finds no backed block waits +//! ([`Answer::Wait`]) and keeps its counter. A re-initialization ([`Ledger::reinit`]) returns +//! the posted count to the initial size and keeps the backing. //! //! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. @@ -52,6 +54,13 @@ pub(crate) const MAX_REFUSALS: u32 = 2; /// Blocks of each PB backing chunk (16 MiB). pub(crate) const CHUNK_BLOCKS: u32 = 128; +/// Backed blocks kept above the posted count: at 1-2 ms per request of ten blocks, 13 requests +/// of lead for the allocation of the next chunk. +pub(crate) const RESERVE_BLOCKS: u32 = CHUNK_BLOCKS; + +/// Chunks of the whole heap. +pub(crate) const MAX_CHUNKS: u32 = MAX_BLOCKS / CHUNK_BLOCKS; + /// The answer to an admitted growth request. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum Answer { @@ -64,24 +73,6 @@ pub(crate) enum Answer { Wait, } -/// The backing chunk that follows `backed` blocks of a ledger with maximum `max`: its first block -/// and size, at most [`CHUNK_BLOCKS`] and never past `max`; none from `max` on. -pub(crate) const fn next_chunk(backed: u32, max: u32) -> Option<(u32, u32)> { - if backed >= max { - None - } else if max - backed < CHUNK_BLOCKS { - Some((backed, max - backed)) - } else { - Some((backed, CHUNK_BLOCKS)) - } -} - -/// The chunks that back a ledger with `initial` and `max` blocks at most: the initial one and -/// [`next_chunk`]'s from `initial` to `max`. -pub(crate) const fn max_chunks(initial: u32, max: u32) -> usize { - 1 + max.saturating_sub(initial).div_ceil(CHUNK_BLOCKS) as usize -} - /// The most blocks a ledger posts: the mapped heap, and below K, so that the block_ctl+4 /// producer (posted modulo K) is never 0, the value of an empty block list. pub(crate) const MAX_POSTED: u32 = if MAX_BLOCKS < BLOCK_LIST_RECORDS { @@ -125,11 +116,32 @@ impl Ledger { self.posted % BLOCK_LIST_RECORDS } + /// Records one more backed chunk. + pub(crate) const fn back_chunk(&mut self) { + self.backed += CHUNK_BLOCKS; + } + /// Backed blocks above the posted count. pub(crate) const fn spare(&self) -> u32 { self.backed.saturating_sub(self.posted) } + /// Whether another chunk is due: fewer than the reserve above the posted count and the heap + /// not yet backed whole. + pub(crate) const fn refill_due(&self) -> bool { + self.backed < MAX_BLOCKS && self.spare() < RESERVE_BLOCKS + } + + /// Chunks that `blocks` posted blocks and the reserve need. + pub(crate) const fn chunks_for(blocks: u32) -> u32 { + let n = (blocks + RESERVE_BLOCKS).div_ceil(CHUNK_BLOCKS); + if n > MAX_CHUNKS { + MAX_CHUNKS + } else { + n + } + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -148,12 +160,6 @@ impl Ledger { && request_epoch(request) == epoch } - /// The blocks the next accepted request posts with the backing it needs: at most - /// [`GROW_STEP`], none at the maximum. - pub(crate) fn next_count(&self) -> u32 { - GROW_STEP.min(self.max - self.posted) - } - /// Accepts an admitted request. Granted (at most [`GROW_STEP`] blocks, and no more than are /// backed) or refused at the maximum, the next expected counter follows it. A request that /// finds no backed block waits: its counter is not consumed, and it is accepted again once a @@ -387,6 +393,105 @@ mod tests { assert_eq!(l.post(10), 10); } + #[test] + fn backing() { + // 64 initial blocks: two chunks (the initial posting and the reserve). + assert_eq!(Ledger::chunks_for(64), 2); + assert_eq!(Ledger::chunks_for(1024), 9); + assert_eq!(Ledger::chunks_for(4095), MAX_CHUNKS); + assert_eq!(MAX_CHUNKS, 32); + let mut l = Ledger::new(true, 64, 4095); + assert_eq!(l.spare(), 0); + assert!(l.refill_due()); + // Nothing backed: a request waits and the counter does not advance. + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + l.back_chunk(); + l.back_chunk(); + assert_eq!((l.backed, l.spare()), (256, 192)); + assert!(!l.refill_due()); + // Granting until the reserve line, then a refill is due; at the end of the backing the + // grant is clipped and the next request waits. + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 10 + } + ); + assert_eq!(l.post(10), 74); + assert_eq!(l.next_request, 1); + l.posted = 128; + assert!(!l.refill_due()); + l.posted = 129; + assert!(l.refill_due()); + l.posted = 250; + assert_eq!( + l.accept(&request(3, 1, 1)), + Answer::Grant { + first: 250, + count: 6 + } + ); + l.post(6); + assert_eq!(l.accept(&request(3, 2, 1)), Answer::Wait); + // The heap end: no refill past the whole heap, the maximum refuses. + l.backed = MAX_BLOCKS; + l.posted = 4090; + assert!(!l.refill_due()); + assert_eq!( + l.accept(&request(3, 2, 1)), + Answer::Grant { + first: 4090, + count: 5 + } + ); + l.post(5); + assert_eq!(l.accept(&request(3, 3, 1)), Answer::Refuse); + // Re-initialization keeps the backing. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 4096)); + } + + /// Every ledger the constructor makes, the driver's included, backed a chunk at a time + /// whenever the reserve is short: no grant posts a block past the backing, no request waits, + /// and the backing ends within the heap. + #[test] + fn backing_ahead_of_every_grant() { + let values = [ + 0, 1, MIN_BLOCKS, 65, 1023, 1024, 1025, 3072, 4094, MAX_POSTED, 5000, + ]; + for grow in [true, false] { + for blocks in values { + for max_blocks in values { + let mut l = Ledger::new(grow, blocks, max_blocks); + for _ in 0..Ledger::chunks_for(l.initial) { + l.back_chunk(); + } + let mut counter = 0; + loop { + match l.accept(&request(5, counter, 1)) { + Answer::Grant { first, count } => { + assert_eq!(first, l.posted); + assert!(count >= 1 && first + count <= l.backed); + l.post(count); + } + Answer::Refuse => break, + Answer::Wait => panic!("waited at {} of {}", l.posted, l.backed), + } + counter += 1; + while l.refill_due() { + l.back_chunk(); + } + } + assert_eq!(l.posted, l.max); + assert!(l.max <= l.backed && l.backed <= MAX_BLOCKS); + } + } + } + } + #[test] fn grants_only_backed_blocks() { let mut l = Ledger::new(true, 64, 4095); diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index 41cc7e438a6d65..e8e49adea03110 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -10,8 +10,9 @@ //! ([`object::Objects::clients`]) as driver mappings at their fixed GVAs, and reserve //! [`super::hw::client::CLIENT_APERTURE`], the pass-slot range and the PB page list //! (`vm.install_driver_mappings`): user binds there fail (EINVAL). The aperture holds the -//! whole PB range, whose backing is mapped later, before the VM's first prefix is published -//! (`Firmware::map_pb`, counted in [`VmState::pb_chunks`]); +//! whole PB range, whose backing chunks are mapped later, before the VM's prefixes are +//! published (`Firmware::map_pb`) and, for the VM of the flights, when a chunk is backed +//! ([`VmState::map_heap`], [`VmState::map_chunk`]); //! - bind it to an exclusive TTBAT slot ([`mmu::Uat::bind`]) for as long as its [`VmState`] //! lives; at most [`super::hw::sizes::MAX_VMS`] such VMs (ENOSPC beyond, checked before //! binding so the slot allocator never blocks). @@ -29,7 +30,7 @@ //! alike. use core::ops::Range; -use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicU32, Ordering}; use kernel::{ drm::gem::BaseObject, new_mutex, @@ -104,6 +105,20 @@ impl ClientAlias { } } +/// A PB heap chunk mapped in a VM at its fixed GVA ([`super::pools::chunk_gva`]). +struct HeapMapping { + gem: ARef, + _mapping: mmu::KernelMapping, +} + +/// The PB heap chunks mapped in a VM, in heap order (chunk `i` at `chunk_gva(i)`). Changed +/// under the device lock (refill, publication). +#[pin_data] +struct HeapMappings { + #[pin] + list: Mutex>, +} + /// G15 state of one user VM, shared by the registry, its queues and its jobs. pub(crate) struct VmState { /// The `mmu::Vm` identifier (`Gpu::ids.vm`). @@ -117,9 +132,8 @@ pub(crate) struct VmState { /// The exclusive slot binding, released before the live count drops. bind: Option, registry: Arc, - /// PB backing chunks mapped into the VM, from the first (`Firmware::map_pb`); changed only - /// under the device lock. - pub(crate) pb_chunks: AtomicUsize, + /// The PB heap chunks mapped in this VM. + heap: Arc, } impl VmState { @@ -127,6 +141,62 @@ impl VmState { pub(crate) fn prune_registry(&self) { self.registry.prune(); } + + /// Maps PB heap chunk `i`, the next one this VM lacks, at its GVA. The alias is held only + /// once it is in the list. + fn map_one(&self, heap: &mut KVec, i: usize, gem: ARef) -> Result { + if i != heap.len() { + return Err(EINVAL); + } + heap.reserve(1, GFP_KERNEL)?; + let mapping = self.vm.map_at( + super::pools::chunk_gva(i), + super::pools::CHUNK_BYTES, + &gem, + mmu::Prot::from_pte(hw::pte::LEAF_CACHED), + false, + )?; + heap.push_within_capacity(HeapMapping { + gem, + _mapping: mapping, + }) + .map_err(|_| ENOMEM) + } + + /// Maps the backed chunks (`gems`, in heap order) this VM does not hold yet. A chunk whose + /// object changed since (freed and backed again) is unmapped first, with everything above + /// it, and so are chunks past the backing. + pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result { + let mut heap = self.heap.list.lock(); + let mut n = 0; + for (i, gem) in gems.enumerate() { + n = i + 1; + if let Some(have) = heap.get(i) { + if core::ptr::eq(&*have.gem, &*gem) { + continue; + } + heap.truncate(i); + } + self.map_one(&mut heap, i, gem)?; + } + heap.truncate(n); + Ok(()) + } + + /// Maps the newly backed chunk `i` into this VM (the VM of a flight, at a refill). + pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result { + let mut heap = self.heap.list.lock(); + if i < heap.len() { + // Already mapped (a flight of the same VM came first). + return Ok(()); + } + self.map_one(&mut heap, i, gem) + } + + /// Unmaps the heap chunks from index `keep` on. + pub(crate) fn unmap_heap_above(&self, keep: usize) { + self.heap.list.lock().truncate(keep); + } } impl Drop for VmState { @@ -292,6 +362,13 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { let status = vm.status().ok_or(EINVAL)?.clone(); let (mappings, reserved) = Registry::driver_mappings(&vm, aliases)?; vm.install_driver_mappings(mappings, reserved)?; + // No PB chunk yet: they are mapped before the VM's first prefix (`Firmware::map_pb`). + let heap = Arc::pin_init( + pin_init!(HeapMappings { + list <- new_mutex!(KVec::new(), "g15::HeapMappings::list"), + }), + GFP_KERNEL, + )?; // The context names the root only now: everything above is already visible to the walker. let bind = gpu.uat.bind(&vm)?; let slot = bind.slot(); @@ -305,7 +382,7 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { vm: vm.clone(), bind: Some(bind), registry: registry.clone(), - pb_chunks: AtomicUsize::new(0), + heap, }, GFP_KERNEL, )?; diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 78cca50e19e109..aa321938476065 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -3,16 +3,17 @@ """Run the G15 PB growth answer and its on-demand backing, actual bodies, against a model. The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), -answer_request(), map_pb(), running_vm() and back_pb(), Pools::soft_max() and backed(), -PbChunk and Pools::new's PB backing statements; pb_max_blocks() from params.rs; the PB sizes, -tuning and PB leaf word from hw.rs; Prot::from_pte() and is_cached_noncoherent() from -pgtable.rs; and the parameter's default from asahi_core.rs. It includes pools/growth.rs, -fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, -the GEM backing allocation, the shared layer's map_at() and append_driver_mappings(), and the -DEVCTL wait, which logs every allocation, alias map, append, unmap, free, block record, -barrier, producer store, reply and log line in order. Allocation, alias map and append -failures can be injected. Source checks: publish_prefix_into() starts with the PB catch-up, -and pools.rs asserts the PB leaf GPU-only at compile time. +answer_request(), map_pb() and refill_backing(), Pools::soft_max(), map_heap() and +back_chunk(), PbChunk, chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from +vm.rs VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() from +params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and +is_cached_noncoherent() from pgtable.rs; and the parameter's default from asahi_core.rs. It +includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model +of the firmware objects, the GEM backing allocation, the shared layer's map_at() and the DEVCTL +wait, which logs every allocation, alias map, unmap, free, block record, barrier, producer +store, reply and log line in order. Allocation and alias map failures can be injected. Source +checks: publish_prefix_into() starts with the PB catch-up, and pools.rs asserts the PB leaf +GPU-only at compile time. What it shows: - a poll pass answers only the requests queued when it starts, in order; one the reply's wait @@ -21,22 +22,24 @@ offset, each producer store follows its records and a barrier, each reply follows its producer store and a barrier, and every producer is in 1..=4095, ending at 4095; - at the maximum two refusals are answered; further requests get no reply, count as bad and - warn once until the ledger is made again, as init() does; + warn once until the ledger is reset, as init() does; - the soft maximum is the parameter clamped to 64..=4095 and then to the ledger's initial..=max; growth answers above it are logged at info level, and the one that reaches it exactly is not; - invalid requests (type, slot, stale counter, epoch, no live render) get no reply, and requests rejected so allocate nothing; -- the backing: the initial blocks only when the GPU starts; chunks of at most 16 MiB, allocated - and held by the running VM before any record or producer names their blocks, never past the - maximum, at most growth::max_chunks(); a failed allocation, alias map or append posts and - answers nothing, leaves the request counter and the VM's count alone and drops what is not - held, and the next request is answered; a failed reply keeps everything; -- a VM's count always equals the aliases it holds, contiguous from the heap's start; a VM that - runs after another grew the backing maps it first (at publication, and again before it grows); - a partial catch-up counts only held aliases; -- the one-VM rule: a catch-up with a flight of another VM, or a count past the chunks, and a - grant with flights of two VMs, are refused (EPROTO) before anything is mapped or posted; +- the backing: the chunks of the initial posting and one reserve chunk when the GPU starts; + chunks of 16 MiB, backed after the replies of a pass (never between them) while fewer than a + reserve chunk remain, mapped into the VM of the flights before any record or producer names + their blocks, never past the heap; with the reserve dry a request waits at the head of the + queue, keeps its counter and is answered once a chunk is backed; a chunk that cannot be mapped + into the running VM is dropped again; a pending re-initialization backs nothing while no + render is published, and still backs for a render published before it; a failed reply keeps + everything; +- a VM's aliases are the chunks, contiguous from the heap's start; a VM that runs after another + grew the backing maps it first (at publication); a partial catch-up keeps the aliases made; +- the one-VM rule: a catch-up with a flight of another VM is refused (EPROTO) before anything + is mapped; - the ledger reset of Pools::init() re-posts from the backing without allocating; - the PB leaf decodes as GPU-only, not cached-noncoherent: dropping an alias is a plain unmap; - Ledger::new() is Ledger::new() as of dba2a110de00 (the carried upstream constructor), capped at @@ -47,8 +50,7 @@ refusal or an unanswered request. The model's VMs have no open/closed state: that a closed VM whose jobs still run keeps its aliases comes from the VM lifetime source (Job and VmJobGuard hold the VM), not from these tests. The ledger reset is not a recovery. The shared layer's -mapping and its rollback are modelled, not run. Running only running_vm()'s or only map_pb()'s -one-VM check on a grant is not distinguishable: each covers it. +mapping is modelled, not run. With --mutations, each mutation below must fail its intended test with the expected message, and the control (a driver maximum of the whole 4096-block heap) must pass, held by the ledger's cap. @@ -73,6 +75,7 @@ HW = f"{G15}/hw.rs" CORE = "drivers/gpu/drm/asahi/asahi_core.rs" ENGINE = f"{G15}/engine.rs" +VM = f"{G15}/vm.rs" PGTABLE = "drivers/gpu/drm/asahi/pgtable.rs" OLD_REF = "dba2a110de00743eeecd41edf23e54ede4491d5e" @@ -95,12 +98,13 @@ } }''' -# Pools::new's PB backing, run verbatim by the model. -POOLS_NEW_BACKING = (' // Backing for the initial blocks; growth adds the rest.\n', ' ledger.backed = ') +# Pools::new's PB backing, run verbatim by the model (up to its log line). +POOLS_NEW_BACKING = (' // Backing for the initial blocks and the reserve; growth adds the rest.\n', + ' dev_info!(\n') # The compile-time proof that a PB alias drop is a plain unmap. PB_LEAF_ASSERT = 'static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent());' -# The two statements of Pools::init() that start a new ledger. +# The two statements of Pools::init() that reset the ledger. LEDGER_RESET = ['self.ledger.reinit();', 'self.exhausted_logged = false;'] @@ -110,9 +114,9 @@ # Leave the last request queued before the pass for later. 'skip-last-queued': (POOLS, 'for _ in 0..queued {', 'for _ in 0..queued.saturating_sub(1) {'), # End the pass at a request that cannot be processed. - 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', - ' self.pools.stats.bad += 1;\n return Ok(());\n' - ' dev_err!('), + 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', + ' self.pools.stats.bad += 1;\n return Ok(());\n' + ' dev_err!('), # Answer three refusals at the maximum. 'three-refusals': (GROWTH, 'pub(crate) const MAX_REFUSALS: u32 = 2;', 'pub(crate) const MAX_REFUSALS: u32 = 3;'), # Answer refusals for ever. @@ -138,42 +142,33 @@ 'let nb = clamp(blocks, MIN_BLOCKS, MAX_BLOCKS);\n' ' (nb, clamp(max_blocks, nb, MAX_BLOCKS))\n' ' } else {\n (MAX_BLOCKS, MAX_BLOCKS)'), - # Post blocks without backing them first (the ledger told the whole heap is backed). - 'grant-without-backing': (POOLS, ' self.back_pb(end, ctx)?;\n self.pools.ledger.backed = self.pools.backed();\n', - ' self.pools.ledger.backed = buffer::MAX_BLOCKS;\n'), - # Back the blocks only after the request is accepted. - 'back-after-accept': [(POOLS, ' self.back_pb(end, ctx)?;\n self.pools.ledger.backed = self.pools.backed();\n', - ' self.pools.ledger.backed = buffer::MAX_BLOCKS;\n'), - (POOLS, ' growth::Answer::Wait => return Err(ENOSPC),\n };\n', - ' growth::Answer::Wait => return Err(ENOSPC),\n };\n self.back_pb(end, ctx)?;\n')], - # Count a catch-up alias before the VM holds it. - 'catch-up-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n mapped += 1;\n' - ' vm.pb_chunks.store(mapped, Ordering::Relaxed);', - ' mapped += 1;\n vm.pb_chunks.store(mapped, Ordering::Relaxed);\n' - ' vm.vm.append_driver_mappings(aliases)?;'), - # Count a growth chunk's alias before the VM holds it. - 'growth-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.', - ' vm.pb_chunks.store(self.pools.chunks.len() + 1, Ordering::Relaxed);\n' - ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.'), - # Keep a chunk whose alias the VM never held. - 'keep-unappended-chunk': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.\n' - ' self.pools\n .chunks\n .push_within_capacity(chunk)\n' - ' .map_err(|_| EIO)?;', - ' // Within the capacity checked above.\n self.pools\n .chunks\n' - ' .push_within_capacity(chunk)\n .map_err(|_| EIO)?;\n' - ' vm.vm.append_driver_mappings(aliases)?;'), - # Grow without catching the running VM up first. - 'no-growth-catch-up': (POOLS, ' let vm = self.running_vm(ctx)?;\n self.map_pb(&vm)?;\n', - ' let vm = self.running_vm(ctx)?;\n'), + # Grant without the backing: neither wait nor clip. + 'grant-without-backing': [(GROWTH, ' if room != 0 && self.spare() == 0 {\n return Answer::Wait;\n }\n', ''), + (GROWTH, 'count: GROW_STEP.min(room).min(self.spare()),', 'count: GROW_STEP.min(room),')], + # Allocate before the pass's answers. + 'refill-before-answers': [(POOLS, ' self.refill_backing();\n Ok(())', ' Ok(())'), + (POOLS, ' let queued = self.events.grow.len();\n', + ' self.refill_backing();\n let queued = self.events.grow.len();\n')], + # Back a chunk without mapping it into the running VM. + 'refill-maps-no-flight': (POOLS, ' flight.job.vm.map_chunk(i, gem.clone())?;\n', + ' let _ = (flight, i, gem.clone());\n'), + # Drop a request that waits for backing. + 'no-wait-requeue': (POOLS, ' if self.events.grow.insert_within_capacity(0, request).is_err() {\n' + ' self.pools.stats.bad += 1;\n }\n', ''), + # Keep a chunk that the running VM could not map. + 'refill-failure-keeps-chunk': (POOLS, ' self.pools.chunks.truncate(backed);\n', ''), + # Back chunks while a re-initialization is pending. + 'refill-during-reinit': (POOLS, ' if self.recovery.reinit && !render {\n return;\n }\n', ''), + # Skip the refill on any pending re-initialization, also with a live render (a stall). + 'refill-skip-with-live-render': (POOLS, 'if self.recovery.reinit && !render {', 'if self.recovery.reinit {'), + # Refill for any live flight, also compute-only ones that cannot ask for growth. + 'refill-for-any-flight': (POOLS, 'if self.recovery.reinit && !render {', + 'if self.recovery.reinit && self.engine.flights.is_empty() {'), # Catch up without the one-VM check. - 'no-activation-guard': (POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), - # Neither the catch-up's nor the grant's one-VM check (each alone covers a grant). - 'no-same-vm-checks': [(POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), - (POOLS, 'if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) {', - 'if vm.slot != ctx {')], + 'no-activation-guard': (POOLS, '.filter(|f| !core::ptr::eq(&*f.job.vm, vm))', '.filter(|_| false)'), # Back the whole heap when the GPU starts. - 'initial-backs-whole-heap': (POOLS, 'chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?;', - 'chunks.push(PbChunk::new(dev, 0, buffer::MAX_BLOCKS)?, GFP_KERNEL)?;'), + 'initial-backs-whole-heap': (POOLS, 'for _ in 0..Ledger::chunks_for(ledger.initial) {', + 'for _ in 0..growth::MAX_CHUNKS {'), # Growth chunks of 32 MiB. 'chunks-above-16mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 128;', 'pub(crate) const CHUNK_BLOCKS: u32 = 256;'), # A firmware-visible cached PB leaf (its drop would ask the firmware to flush). @@ -183,7 +178,7 @@ # The test each mutation must fail, and a piece of its failure message. INTENDED = { - 'drain-until-empty': ('queued_snapshot_keeps_order', 'unbounded answering in one poll pass'), + 'drain-until-empty': ('queued_snapshot_keeps_order', 'replies in the first pass'), 'skip-last-queued': ('queued_snapshot_keeps_order', 'replies in the first pass'), 'return-after-request-error': ('unprocessed_request_does_not_end_the_pass', 'requests taken in the pass'), 'three-refusals': ('default_growth_to_exhaustion', 'want Some(Log("warn"'), @@ -195,14 +190,15 @@ 'soft-boundary-inclusive': ('soft_max_boundaries', 'got Some(Log("info", "")), want Some(Log("dbg", ""))'), 'soft-max-unclamped': ('soft_max_bounds', 'soft_max, param 0'), 'uncapped-ledger': ('constructor_is_the_old_one_capped', 'new(true, '), - 'grant-without-backing': ('growth_backs_before_posting', 'record of block'), - 'back-after-accept': ('allocation_failure_posts_nothing', 'ledger after a backing failure'), - 'catch-up-cursor-before-append': ('partial_catch_up_counts_only_held_aliases', 'count after a failed append'), - 'growth-cursor-before-append': ('append_failure_posts_nothing', 'count differs from held aliases'), - 'keep-unappended-chunk': ('append_failure_posts_nothing', 'chunks after a backing failure'), - 'no-growth-catch-up': ('growth_catches_the_running_vm_up_first', 'running VM not caught up'), + 'grant-without-backing': ('dry_reserve_waits_and_keeps_order', 'record of block'), + 'refill-before-answers': ('allocation_follows_the_replies', 'allocation before a reply'), + 'refill-maps-no-flight': ('growth_backs_before_posting', 'record of block'), + 'no-wait-requeue': ('dry_reserve_waits_and_keeps_order', 'waiting request lost'), + 'refill-failure-keeps-chunk': ('refill_map_failure_drops_the_chunk', 'chunk kept after a failed map'), + 'refill-during-reinit': ('pending_reinit_without_flights_backs_nothing', 'backed during a pending re-initialization'), + 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), + 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), - 'no-same-vm-checks': ('grant_refuses_flights_of_two_vms', 'grant with flights of two VMs'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), 'pb-leaf-fw-visible': ('pb_leaf_is_gpu_only', 'PB leaf not GPU-only'), @@ -292,6 +288,26 @@ def consts(text, names): pub fn last(&self) -> Option<&T> { self.0.last() } + pub fn truncate(&mut self, n: usize) { + self.0.truncate(n) + } + pub fn reserve(&mut self, n: usize, _: Flags) -> Result { + self.0.reserve(n); + Ok(()) + } + pub fn iter(&self) -> std::slice::Iter<'_, T> { + self.0.iter() + } +} +/// A kernel mutex, single-threaded here. +pub struct Mutex(pub RefCell); +impl Mutex { + pub fn new(value: T) -> Self { + Mutex(RefCell::new(value)) + } + pub fn lock(&self) -> std::cell::RefMut<'_, T> { + self.0.borrow_mut() + } } #[macro_export] macro_rules! static_assert { @@ -311,20 +327,17 @@ def consts(text, names): /// Backing object `id` of this many bytes allocated, or freed (its last reference gone). Alloc(u32, usize), Free(u32), - /// A PB alias of object `id` at GVA in the VM with this context, made, held by the VM - /// (appended to its driver mappings), or unmapped. + /// A PB alias of object `id` at GVA in the VM with this context, made or unmapped. Map(u32, u64, usize, u32), - Append(u32, u64), Unmap(u32, u64), } thread_local! { pub static LOG: RefCell> = RefCell::new(Vec::new()); pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); - /// Failures to inject: the next this many allocations, alias maps, appends fail. + /// Failures to inject: the next this many allocations or alias maps fail. pub static FAIL_ALLOC: Cell = Cell::new(0); pub static FAIL_MAP: Cell = Cell::new(0); pub static MAP_OK_BEFORE_FAIL: Cell = Cell::new(0); - pub static FAIL_APPEND: Cell = Cell::new(0); pub static NEXT_ID: Cell = Cell::new(1); } /// Takes one injected failure of `which`. @@ -391,7 +404,7 @@ def consts(text, names): } } -/// The shared layer's VM: kernel aliases at fixed addresses, held by the VM once appended. +/// The shared layer's VM: kernel aliases at fixed addresses. pub mod mmu { use crate::prelude::*; pub struct Prot(pub u64); @@ -414,11 +427,10 @@ def consts(text, names): } pub struct Vm { pub slot: u32, - pub held: std::cell::RefCell>, } impl Vm { pub fn new(slot: u32) -> Self { - Vm { slot, held: Default::default() } + Vm { slot } } pub fn map_at( &self, @@ -439,24 +451,9 @@ def consts(text, names): if !skip && crate::fail(&crate::FAIL_MAP) { return Err(ENOMEM); } - // Leaves already in use: EBUSY (prepare_map). - let end = addr + size as u64; - if self.held.borrow().iter().any(|m| m.gva < end && addr < m.gva + m.size as u64) { - return Err(EBUSY); - } crate::ev(crate::Ev::Map(self.slot, addr, size, gem.id)); Ok(KernelMapping { slot: self.slot, gva: addr, size, prot: prot.0, gem: gem.clone() }) } - pub fn append_driver_mappings(&self, mappings: KVec) -> Result { - if crate::fail(&crate::FAIL_APPEND) { - return Err(ENOMEM); // the mappings drop here - } - for m in mappings.0 { - crate::ev(crate::Ev::Append(self.slot, m.gva)); - self.held.borrow_mut().push(m); - } - Ok(()) - } } } ''' @@ -479,14 +476,26 @@ def consts(text, names): pub struct Init { pub control: Option>, } + /// vm.rs's chunk mapping, its fields as the driver names them. + pub struct HeapMapping { + pub gem: ARef, + pub _mapping: mmu::KernelMapping, + } + pub struct HeapMappings { + pub list: Mutex>, + } pub struct VmState { pub slot: u32, pub vm: mmu::Vm, - pub pb_chunks: AtomicUsize, + pub heap: Arc, } impl VmState { pub fn new(slot: u32) -> Arc { - Arc::new(VmState { slot, vm: mmu::Vm::new(slot), pb_chunks: AtomicUsize::new(0) }) + Arc::new(VmState { + slot, + vm: mmu::Vm::new(slot), + heap: Arc::new(HeapMappings { list: Mutex::new(KVec(Vec::new())) }), + }) } } pub struct Job { @@ -498,10 +507,13 @@ def consts(text, names): pub struct Engine { pub flights: Vec, } + pub struct Recovery { + pub reinit: bool, + } pub struct Receipts { pub renders: u32, } - /// A firmware object: the block list, block_ctl or the heap. + /// A firmware object: the block list or block_ctl. pub struct FwObject { pub gva: Option, pub size: usize, @@ -534,6 +546,8 @@ def consts(text, names): pub grown: u64, pub refused: u64, pub bad: u64, + pub waits: u64, + pub refills: u64, } pub struct Live { pub block_ctl: FwObject, @@ -554,6 +568,10 @@ def consts(text, names): pub fn chunk_capacity(&self) -> usize { self.chunks.capacity() } + /// The backing objects' ids, in heap order. + pub fn chunk_ids(&self) -> Vec { + self.chunks.iter().map(|c| c.backing.gem.id).collect() + } } pub struct Queue(pub std::collections::VecDeque<[u32; 14]>); impl Queue { @@ -567,6 +585,11 @@ def consts(text, names): assert_eq!(i, 0); self.0.pop_front().ok_or(()) } + pub fn insert_within_capacity(&mut self, i: usize, q: [u32; 14]) -> core::result::Result<(), [u32; 14]> { + assert_eq!(i, 0); + self.0.push_front(q); + Ok(()) + } } pub struct Events { pub grow: Queue, @@ -599,6 +622,7 @@ def consts(text, names): pub pools: Pools, pub init: Init, pub engine: Engine, + pub recovery: Recovery, pub renders: Vec, pub fw: FwModel, } @@ -614,7 +638,7 @@ def consts(text, names): crate::take(); f } - /// Pools::new's PB backing: the initial chunk, room for every chunk the ledger needs. + /// Pools::new's PB backing: the initial chunks and the reserve, room for the heap. pub fn without_flights() -> Self { let drm = ARef::new(AsahiDevice); let (ledger, chunks) = pools_new_backing(&drm).unwrap(); @@ -637,6 +661,7 @@ def consts(text, names): }, init: Init { control: Some(std::sync::Arc::new(Ctl { epoch: 0x1_0000_0003 })) }, engine: Engine { flights: Vec::new() }, + recovery: Recovery { reinit: false }, renders: vec![1], fw: FwModel { ctx: 5, @@ -693,10 +718,10 @@ def consts(text, names): use crate::g15::firmware::{Firmware, VmState, Flight, Job, HEAP}; use crate::g15::hw::{client, pte}; use crate::g15::fw::buffer::{BLOCK_BYTES, MAX_BLOCKS}; - use crate::g15::pools::growth::{max_chunks, CHUNK_BLOCKS}; + use crate::g15::pools::growth::{CHUNK_BLOCKS, MAX_CHUNKS}; + use crate::g15::pools::{chunk_gva, CHUNK_BYTES}; use crate::g15::pgtable::Prot; - use crate::{EPROTO, FAIL_ALLOC, FAIL_APPEND, FAIL_MAP}; - use core::sync::atomic::Ordering; + use crate::{EPROTO, FAIL_ALLOC, FAIL_MAP}; use std::sync::Arc; use crate::g15::fw::buffer::{self, grow_reply, BLOCK_LIST_RECORDS, GROW_STEP}; use crate::g15::fw::channels::devctl; @@ -777,7 +802,7 @@ def consts(text, names): fn levels_only(evs: Vec) -> Vec { evs.into_iter() - .filter(|e| !is_backing(e)) + .filter(|e| !is_backing(e) && !matches!(e, Ev::Log(_, m) if m.starts_with("PB heap:"))) .map(|e| match e { Ev::Log(l, _) => Ev::Log(l, String::new()), e => e, @@ -890,6 +915,7 @@ def consts(text, names): f.queue(); f.queue(); take(); + let mapped = f.pools.ledger.backed as u64; f.pass(); let log = take(); let replies: Vec = log @@ -981,7 +1007,7 @@ def consts(text, names): assert_eq!(log.len(), 4); assert_eq!(f.pools.stats.bad, 4); let mut fresh = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); - fresh.backed = fresh.initial; + fresh.backed = Ledger::chunks_for(fresh.initial) * CHUNK_BLOCKS; assert_eq!(f.pools.ledger, fresh); // No render published: not live. f.renders[0] = 0; @@ -1000,54 +1026,43 @@ def consts(text, names): } fn is_backing(e: &Ev) -> bool { - matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Append(..) | Ev::Unmap(..)) + matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Unmap(..)) } - /// A VM's PB aliases: its count, and the aliases it holds, from block 0 in order. - fn assert_cursor(vm: &VmState, chunks: usize) { - let held = vm.vm.held.borrow(); - assert_eq!(vm.pb_chunks.load(Ordering::Relaxed), held.len(), "count differs from held aliases"); - assert!(held.len() <= chunks); - let mut gva = client::PB_HEAP; - for m in held.iter() { - assert_eq!(m.gva, gva, "aliases not contiguous from the heap's start"); - assert_eq!(m.prot, pte::LEAF_CACHED); - gva += m.size as u64; + /// A VM's PB aliases: exactly the first `chunks` chunks, in order, each the whole chunk at + /// its GVA with the PB leaf, each of the chunk's object. + fn assert_cursor(f: &Firmware, vm: &VmState, chunks: usize) { + let heap = vm.heap.list.lock(); + assert_eq!(heap.len(), chunks, "VM aliases differ from the chunks"); + let ids = f.pools.chunk_ids(); + for (i, m) in heap.iter().enumerate() { + assert_eq!(m._mapping.gva, chunk_gva(i), "aliases not contiguous from the heap's start"); + assert_eq!(m._mapping.size, CHUNK_BYTES); + assert_eq!(m._mapping.prot, pte::LEAF_CACHED); + assert_eq!(m.gem.id, ids[i], "alias of another object"); } } /// In the order of `log`: no block record and no producer past the blocks the VM with - /// context `slot` holds aliases for at that point. + /// context `slot` has mapped at that point (`start` blocks before the log). fn assert_posted_backed(log: &[Ev], slot: u32, start: u64) { let mut mapped = start; for e in log { match e { - Ev::Append(s, gva) if *s == slot => { - assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "append out of order"); + Ev::Map(s, gva, size, _) if *s == slot => { + assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "map out of order"); + mapped += *size as u64 / BLOCK_BYTES; } - Ev::Map(s, gva, size, _) if *s == slot && *gva == client::PB_HEAP + mapped * BLOCK_BYTES => { - // Counted at its append. - let _ = size; + Ev::Unmap(s, gva) if *s == slot => { + mapped = mapped.min((gva - client::PB_HEAP) / BLOCK_BYTES); } - _ => {} - } - if let Ev::Append(s, gva) = e { - if *s == slot { - let size = log - .iter() - .find_map(|m| match m { - Ev::Map(ms, mg, size, _) if ms == s && mg == gva => Some(*size as u64), - _ => None, - }) - .unwrap(); - mapped += size / BLOCK_BYTES; + Ev::Record(off, _) => { + assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); } - } - if let Ev::Record(off, _) = e { - assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); - } - if let Ev::Producer(p) = e { - assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + Ev::Producer(p) => { + assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + } + _ => {} } } } @@ -1073,17 +1088,20 @@ def consts(text, names): let mut f = Firmware::without_flights(); let log = take(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); - assert_eq!(log, vec![Ev::Alloc(1, initial as usize * BLOCK_BYTES as usize)], "startup allocation"); - assert_eq!((f.pools.backed(), f.pools.chunk_count()), (initial, 1)); - assert_eq!(f.pools.chunk_capacity(), max_chunks(initial, max)); - assert_eq!(max_chunks(initial, max), 25); + let chunks = Ledger::chunks_for(initial) as usize; + assert_eq!(chunks, 9); + let want: Vec = (1..=chunks as u32).map(|id| Ev::Alloc(id, CHUNK_BYTES)).collect(); + assert_eq!(log, want, "startup allocation"); + assert_eq!((f.pools.ledger.backed, f.pools.chunk_count()), (chunks as u32 * CHUNK_BLOCKS, chunks)); + assert_eq!(f.pools.chunk_capacity(), MAX_CHUNKS as usize); + assert!(max <= MAX_CHUNKS * CHUNK_BLOCKS); // The first prefix of a VM maps the backing, nothing else. let vm = VmState::new(7); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); f.map_pb(&vm).unwrap(); - let size = initial as usize * BLOCK_BYTES as usize; - assert_eq!(take(), vec![Ev::Map(7, client::PB_HEAP, size, 1), Ev::Append(7, client::PB_HEAP)]); - assert_cursor(&vm, 1); + let want: Vec = (0..chunks).map(|i| Ev::Map(7, chunk_gva(i), CHUNK_BYTES, i as u32 + 1)).collect(); + assert_eq!(take(), want); + assert_cursor(&f, &vm, chunks); // Mapping again maps nothing. f.map_pb(&vm).unwrap(); assert!(take().is_empty()); @@ -1099,6 +1117,8 @@ def consts(text, names): fn growth_backs_before_posting() { PARAM.with(|p| p.set(PARAM_DEFAULT)); let mut f = Firmware::new(); + let start = f.pools.ledger.backed as u64; + let first_chunks = f.pools.chunk_count(); f.fw.ask_after_reply = true; f.queue(); take(); @@ -1109,77 +1129,180 @@ def consts(text, names): assert!(passes <= 1000, "growth did not end"); } let log = take(); - let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); - assert_posted_backed(&log, 5, initial as u64); - // Chunks of at most 128 blocks, allocated only as needed: up to the maximum, no further. + assert_posted_backed(&log, 5, start); + // 16 MiB chunks, backed as the reserve needs them: up to the whole heap, no further. let allocs: Vec = log .iter() .filter_map(|e| if let Ev::Alloc(_, b) = e { Some(*b) } else { None }) .collect(); - assert_eq!(allocs.len() + 1, max_chunks(initial, max)); assert!(allocs.iter().all(|b| *b <= 16 << 20), "a growth chunk above 16 MiB"); assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 16 << 20); - assert_eq!(f.pools.backed(), max); + assert_eq!(allocs.len() + first_chunks, MAX_CHUNKS as usize); + assert_eq!(f.pools.ledger.backed, MAX_BLOCKS); + assert_eq!(f.pools.stats.waits, 0); + assert_eq!(f.pools.stats.refills, allocs.len() as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); - assert_cursor(&f.engine.flights[0].job.vm, f.pools.chunk_count()); + assert_cursor(&f, &f.engine.flights[0].job.vm, f.pools.chunk_count()); } - /// A failure while backing posts nothing and answers nothing; the request counter and the - /// VM's count stay; the next request is answered. - fn failure_posts_nothing(which: &'static std::thread::LocalKey>) -> Vec { + #[test] + fn allocation_follows_the_replies() { let mut f = Firmware::new(); - which.with(|c| c.set(1)); + // Below the reserve line: the pass's answers come first, then the chunk. + f.pools.ledger.posted = f.pools.ledger.backed - growth_reserve() + 1; f.queue(); - f.pass(); - let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_) | Ev::Sync)), 0, - "posted after a backing failure: {log:?}"); - assert_eq!((f.pools.ledger.posted, f.pools.ledger.next_request), (1024, 0), "ledger after a backing failure"); - assert_eq!((f.pools.stats.bad, f.pools.chunk_count()), (1, 1), "chunks after a backing failure"); - assert_cursor(&f.engine.flights[0].job.vm, 1); - assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -12")))); f.queue(); + take(); f.pass(); - let retry = take(); - assert!(retry.contains(&Ev::Producer(1034))); - assert_eq!(f.pools.chunk_count(), 2); - assert_cursor(&f.engine.flights[0].job.vm, 2); - log + let log = take(); + let last_reply = log.iter().rposition(|e| matches!(e, Ev::Reply(_))).expect("replies"); + let first_alloc = log.iter().position(|e| matches!(e, Ev::Alloc(..))).expect("a refill"); + assert!(first_alloc > last_reply, "allocation before a reply: {log:?}"); + assert_eq!(count(&log, |e| matches!(e, Ev::Reply(_))), 2); + assert_eq!(f.pools.stats.refills, 1); + } + + fn growth_reserve() -> u32 { + crate::g15::pools::growth::RESERVE_BLOCKS } #[test] - fn allocation_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_ALLOC); - assert_eq!(count(&log, is_backing), 0); + fn dry_reserve_waits_and_keeps_order() { + let mut f = Firmware::new(); + let start = f.pools.ledger.backed as u64; + FAIL_ALLOC.with(|c| c.set(1000)); + let mut log = Vec::new(); + let mut waited = None; + for _ in 0..40 { + f.queue(); + f.pass(); + let pass = take(); + let replied = pass.iter().any(|e| matches!(e, Ev::Reply(_))); + log.extend(pass); + if !replied { + waited = Some(f.fw.next_counter - 1); + break; + } + } + assert_posted_backed(&log, 5, start); + let waited = waited.expect("no request waited"); + assert_eq!(f.pools.ledger.posted, f.pools.ledger.backed); + let queued: Vec = f.events.grow.0.iter().map(|q| q[buffer::grow_request::COUNTER]).collect(); + assert_eq!(queued, vec![waited], "waiting request lost"); + assert_eq!(f.pools.ledger.next_request, waited); + assert!(f.pools.stats.waits >= 1); + // The allocator recovers: this pass still waits, then backs a chunk; the next answers. + FAIL_ALLOC.with(|c| c.set(0)); + f.pass(); + let log = take(); + assert!(!log.iter().any(|e| matches!(e, Ev::Reply(_)))); + assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 1); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 1); + let mapped = f.pools.ledger.backed as u64; + f.pass(); + let log = take(); + let replies: Vec = log + .iter() + .filter_map(|e| if let Ev::Reply(r) = e { Some(r[grow_reply::COUNTER]) } else { None }) + .collect(); + assert_eq!(replies, vec![waited]); + assert!(f.events.grow.is_empty()); + assert_posted_backed(&log, 5, mapped); } #[test] - fn map_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_MAP); + fn refill_map_failure_drops_the_chunk() { + let mut f = Firmware::new(); + let chunks = f.pools.chunk_count(); + let backed = f.pools.ledger.backed; + f.pools.ledger.posted = backed - growth_reserve() + 1; + FAIL_MAP.with(|c| c.set(1)); + take(); + f.pass(); + let log = take(); + assert_eq!(f.pools.chunk_count(), chunks, "chunk kept after a failed map"); let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); - assert!(matches!(backing[..], [Ev::Alloc(2, _), Ev::Free(2)]), "{backing:?}"); + assert!(matches!(backing[..], [Ev::Alloc(id, _), Ev::Free(free)] if id == free), "{backing:?}"); + assert_eq!(f.pools.ledger.backed, backed); + assert!(log.iter().any(|e| matches!(e, Ev::Log("warn", m) if m.contains("backing chunk")))); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks); + // The next pass backs it. + f.pass(); + assert_eq!(f.pools.chunk_count(), chunks + 1); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks + 1); } + /// A pending re-initialization with nothing published (a reclaim, or a recovery that + /// failed the flights): the posted count is about to drop, so nothing is backed. #[test] - fn append_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_APPEND); - let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); + fn pending_reinit_without_flights_backs_nothing() { + let mut f = Firmware::without_flights(); + f.pools.ledger.posted = f.pools.ledger.backed - growth_reserve() + 1; + f.recovery.reinit = true; + take(); + f.pass(); + assert_eq!(count(&take(), is_backing), 0, "backed during a pending re-initialization"); + f.recovery.reinit = false; + f.pass(); + assert_eq!(f.pools.stats.refills, 1); + } + + /// A pending re-initialization with only compute published (no render can ask for + /// growth): nothing is backed. + #[test] + fn pending_reinit_with_compute_only_backs_nothing() { + let mut f = Firmware::new(); + f.renders[0] = 0; + f.pools.ledger.posted = f.pools.ledger.backed; + f.recovery.reinit = true; + take(); + for _ in 0..4 { + f.pass(); + } + assert_eq!(count(&take(), is_backing), 0, "backed for a compute-only flight"); + assert_eq!(f.pools.stats.refills, 0); + } + + /// A render published before the re-initialization was set (the flights behind a fragment + /// kill) still asks for growth: with its reserve used up, the refill must still back + /// chunks, or the request waits until the deadline. + #[test] + fn pending_reinit_with_a_live_render_refills() { + let mut f = Firmware::new(); // one live render + let start = f.pools.ledger.backed as u64; + f.pools.ledger.posted = f.pools.ledger.backed; + f.recovery.reinit = true; + f.queue(); + take(); + for _ in 0..4 { + f.pass(); + } + let s = &f.pools.stats; assert!( - matches!(backing[..], [Ev::Alloc(2, _), Ev::Map(5, _, _, 2), Ev::Unmap(5, _), Ev::Free(2)]), - "{backing:?}" + f.events.grow.is_empty(), + "growth request stranded: posted={} backed={} waits={} refills={}", + f.pools.ledger.posted, + f.pools.ledger.backed, + s.waits, + s.refills ); + // The request waits one pass for the chunk; the grant's pass tops the reserve up. + assert_eq!((s.waits, s.refills, s.requests), (1, 2, 1)); + assert_posted_backed(&take(), 5, start); } #[test] fn transport_failure_keeps_everything() { let mut f = Firmware::new(); + let chunks = f.pools.chunk_count(); f.fw.reply_fails = true; f.queue(); f.pass(); let log = take(); assert!(log.contains(&Ev::Producer(1034))); - assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (1034, 2)); - assert_cursor(&f.engine.flights[0].job.vm, 2); + // The grant crossed the reserve line: one chunk backed after it. + assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (1034, chunks + 1)); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks + 1); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); } @@ -1189,7 +1312,8 @@ def consts(text, names): let a = f.engine.flights[0].job.vm.clone(); grow_to(&mut f, 1100); take(); - assert_eq!(f.pools.chunk_count(), 2); + let chunks = f.pools.chunk_count(); + assert_eq!(chunks, 10); // A's work retires; a new VM runs next: it maps every chunk first. f.engine.flights.clear(); let b = VmState::new(6); @@ -1197,68 +1321,37 @@ def consts(text, names): f.fw.ctx = 6; f.map_pb(&b).unwrap(); let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Append(6, _))), 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(6, ..))), chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 0); - assert_cursor(&b, 2); + assert_cursor(&f, &b, chunks); // B grows the backing; A, behind, catches up when it runs again. grow_to(&mut f, 1200); let log = take(); - assert_posted_backed(&log, 6, 1152); + assert_posted_backed(&log, 6, chunks as u64 * CHUNK_BLOCKS as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 0); + assert_eq!(f.pools.chunk_count(), chunks + 1); f.engine.flights.clear(); f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); f.map_pb(&a).unwrap(); - assert_cursor(&a, f.pools.chunk_count()); - assert_eq!(a.pb_chunks.load(Ordering::Relaxed), f.pools.chunk_count()); + assert_cursor(&f, &a, f.pools.chunk_count()); } #[test] - fn growth_catches_the_running_vm_up_first() { + fn partial_catch_up_keeps_the_aliases_made() { let mut f = Firmware::new(); - let a = f.engine.flights[0].job.vm.clone(); - // B grows the backing while A is idle. - f.engine.flights.clear(); - let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.map_pb(&b).unwrap(); - f.fw.ctx = 6; grow_to(&mut f, 1100); - // A runs again; its growth request maps what it lacks before backing more. - f.engine.flights.clear(); - f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); - f.fw.ctx = 5; - take(); - grow_to(&mut f, 1130); - let log = take(); - assert_eq!(a.vm.held.borrow().len(), f.pools.chunk_count(), "running VM not caught up"); - assert_cursor(&a, f.pools.chunk_count()); - assert_posted_backed(&log, 5, 1024); - } - - #[test] - fn partial_catch_up_counts_only_held_aliases() { - let mut f = Firmware::new(); - grow_to(&mut f, 1450); + let chunks = f.pools.chunk_count(); f.engine.flights.clear(); let b = VmState::new(6); f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); take(); // The third alias cannot be made: two held. - FAIL_MAP.with(|c| c.set(0)); crate::MAP_OK_BEFORE_FAIL.with(|c| c.set(2)); FAIL_MAP.with(|c| c.set(1)); assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_cursor(&b, 5); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2); - // The third alias is made but cannot be held: dropped, still two. - FAIL_APPEND.with(|c| c.set(1)); - assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2, "count after a failed append"); - assert!(take().iter().any(|e| matches!(e, Ev::Unmap(6, _)))); - assert_cursor(&b, 5); + assert_cursor(&f, &b, 2); f.map_pb(&b).unwrap(); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 5); - assert_cursor(&b, 5); + assert_cursor(&f, &b, chunks); } /// The ledger reset of Pools::init(), not a full recovery. @@ -1274,7 +1367,7 @@ def consts(text, names): let log = take(); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..) | Ev::Map(..))), 0); assert_eq!(f.pools.chunk_count(), chunks); - assert_posted_backed(&log, 5, f.pools.backed() as u64); + assert_posted_backed(&log, 5, f.pools.ledger.backed as u64); } #[test] @@ -1292,33 +1385,11 @@ def consts(text, names): #[test] fn catch_up_refuses_flights_of_another_vm() { let mut f = Firmware::new(); - let a = f.engine.flights[0].job.vm.clone(); let b = VmState::new(6); f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); assert_eq!(f.map_pb(&b), Err(EPROTO), "catch-up with a flight of another VM"); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 0); + assert_eq!(b.heap.list.lock().len(), 0); assert_eq!(count(&take(), is_backing), 0); - // A count past the chunks. - f.engine.flights.pop(); - a.pb_chunks.store(f.pools.chunk_count() + 1, Ordering::Relaxed); - assert_eq!(f.map_pb(&a), Err(EPROTO), "catch-up with a count past the chunks"); - } - - #[test] - fn grant_refuses_flights_of_two_vms() { - let mut f = Firmware::new(); - let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.renders.push(1); - f.queue(); - f.pass(); - let log = take(); - assert_eq!( - count(&log, |e| is_backing(e) || matches!(e, Ev::Reply(_) | Ev::Producer(_) | Ev::Record(..))), - 0, - "grant with flights of two VMs" - ); - assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -71")))); } #[test] @@ -1388,7 +1459,7 @@ def read(path, ref=None): raise SystemExit('embedded Ledger::new() differs from ' + OLD_REF) def run(mutation): - paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE) + paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE, VM) src = {path: read(path) for path in paths} if mutation: spec = {**MUTATIONS, **CONTROLS}[mutation] @@ -1398,15 +1469,23 @@ def run(mutation): src[path] = src[path].replace(old, new) pools, hw = src[POOLS], src[HW] bodies = [item(pools, s) for s in (' pub(crate) fn answer_growth(&mut self) -> Result {', - ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', + ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', ' pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result {', - ' fn running_vm(&self, ctx: u32) -> Result> {', - ' fn back_pb(&mut self, end: u32, ctx: u32) -> Result {')] + ' pub(crate) fn refill_backing(&mut self) {')] soft_max = item(pools, ' pub(crate) fn soft_max(&self) -> u32 {') - backed = item(pools, ' pub(crate) fn backed(&self) -> u32 {') + pools_methods = (item(pools, ' fn map_heap(&self, vm: &VmState) -> Result {') + '\n' + + item(pools, ' fn back_chunk(&mut self) -> Result> {')) chunk = item(pools, 'struct PbChunk {') + '\n' + item(pools, 'impl PbChunk {') + chunk_geometry = (re.search(r'^pub\(crate\) const CHUNK_BYTES: usize = [^;]+;', pools, re.M).group(0) + '\n' + + item(pools, 'pub(crate) const fn chunk_gva(i: usize) -> u64 {')) + vm_src = src[VM] + vm_methods = '\n'.join(item(vm_src, s) for s in ( + ' fn map_one(&self, heap: &mut KVec, i: usize, gem: ARef) -> Result {', + ' pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result {', + ' pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result {', + ' pub(crate) fn unmap_heap_above(&self, keep: usize) {')) start = pools.index(POOLS_NEW_BACKING[0]) - end = pools.index('\n', pools.index(POOLS_NEW_BACKING[1], start)) + 1 + end = pools.index(POOLS_NEW_BACKING[1], start) pools_new_backing = pools[start:end] if pools.count(PB_LEAF_ASSERT) != 1: raise SystemExit('pools.rs does not assert the PB leaf GPU-only') @@ -1440,7 +1519,8 @@ def run(mutation): control = consts(item(src[INITDATA], 'pub(crate) mod control {'), ['HALT_COUNT']) growth = src[GROWTH] clamp_fn = item(growth, 'const fn clamp(v: u32, lo: u32, hi: u32) -> u32 {') - tested = (''.join(bodies) + soft_max + backed + chunk + pools_new_backing + pb_max_blocks + growth + + tested = (''.join(bodies) + soft_max + pools_methods + chunk + chunk_geometry + vm_methods + + pools_new_backing + pb_max_blocks + growth + src[BUFFER] + sizes + tuning + pte + prot) dir = out / (mutation or 'actual') dir.mkdir(exist_ok=True) @@ -1453,7 +1533,8 @@ def run(mutation): f' #[path = "{dir / "buffer.rs"}"] pub mod buffer;\n' ' pub mod initdata { pub mod control {\n' + control + '\n} }\n' ' }\n' - f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth; }}\n' + f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth;\n' + ' use super::fw::buffer;\n use super::hw::client;\n' + chunk_geometry + '\n }\n' ' pub mod hw {\n' ' pub mod sizes {\n' + sizes + '\n}\n' ' pub mod tuning {\n' + tuning + '\n}\n' @@ -1486,19 +1567,20 @@ def run(mutation): ' use super::fw::buffer::{self, block_ctl};\n' ' use super::fw::channels::DevCtl;\n' ' use super::fw::initdata::control;\n' - ' use super::hw::{client, pte, tuning};\n' - ' use super::pools::growth::{self, Ledger};\n' + ' use super::hw::{self, client, pte, tuning};\n' + ' use super::pools::growth::{self, Answer, Ledger};\n' + ' use super::pools::{chunk_gva, CHUNK_BYTES};\n' ' use super::object::{self, HeldPages};\n' ' use crate::driver::AsahiDevice;\n' - ' use crate::{gem, mmu, ARef};\n' - ' use core::sync::atomic::{AtomicUsize, Ordering};\n' + ' use crate::{gem, mmu, ARef, Mutex};\n' ' use std::sync::Arc;\n' + MODEL.replace('p.RESET', reset).replace('POOLS_NEW_BACKING', '') + chunk + '\n' ' /// Pools::new\'s PB backing, verbatim.\n' ' pub fn pools_new_backing(dev: &AsahiDevice) -> Result<(Ledger, KVec)> {\n' + pools_new_backing + ' Ok((ledger, chunks))\n }\n' - ' impl Pools {\n' + soft_max + '\n' + backed + '\n }\n' + ' impl Pools {\n' + soft_max + '\n' + pools_methods + '\n }\n' + ' impl VmState {\n' + vm_methods + '\n }\n' ' impl Firmware {\n' + '\n'.join(bodies) + '\n }\n' ' }\n' '}\n' + TESTS) @@ -1538,7 +1620,8 @@ def no_core(): result = run(a.mutation) receipt = {'ref': a.ref, 'old_ref': OLD_REF, 'old_reference_checked_against_git': old_git is not None, **result, - 'scope': 'Actual answer_growth/answer_request/soft_max/pb_max_blocks, pools/growth.rs, ' + 'scope': 'Actual answer_growth/answer_request/map_pb/refill_backing/soft_max/pb_max_blocks, ' + 'the PB chunk and VM heap-mapping bodies, pools/growth.rs, ' 'fw/buffer.rs and fw/channels.rs over a model of the firmware objects with an ' 'ordered access log, and Ledger::new against its body at ' f'{OLD_REF[:12]}. Not kernel, firmware or GPU execution, timing, or recovery.'} From b91c280b3294d2aa19a7f8d5796528b68412c6b6 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:56 +0530 Subject: [PATCH 05/11] drm/asahi: g15: m3: start the parameter buffer at 64 blocks The initial posting is 1024 blocks (128 MiB), with 9 chunks (144 MiB) backed when the GPU starts. On the J516C, an idle desktop with a 64-block start stayed at 64 posted blocks (8 MiB), and growth is a round trip of 1-2 ms per ten blocks. Post 64 blocks at (re)initialization by default, settable with g15_pb_blocks (64..4095, read-only after boot). The PB then needs two chunks (32 MiB) when the GPU starts: the posting and the reserve. tuning::PB_BLOCKS is removed. g15_pb_max_blocks, the firmware's growth maximum, is still raised to at least the initial posting; its help text no longer says that the 512 MiB heap is allocated in any case. This applies to the T6030 and the T6031, which share this code; it is untested on the T6030, and g15_pb_blocks=1024 restores the old start. The zero-words harness pins hw.rs whole; its hash is refreshed, and its T6030 configuration module stays pinned to the original bytes. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/asahi/asahi_core.rs | 6 +- drivers/gpu/drm/asahi/g15/hw.rs | 7 +-- drivers/gpu/drm/asahi/g15/params.rs | 12 +++- drivers/gpu/drm/asahi/g15/pools.rs | 18 +++--- drivers/gpu/drm/asahi/g15/pools/growth.rs | 6 +- tools/asahi/g15-pb-growth-tests.py | 76 +++++++++++++++-------- tools/asahi/g15-zero-words-tests.py | 8 ++- 7 files changed, 85 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 73a798fc69c00c..084718c5b1dcbe 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -137,7 +137,11 @@ module! { }, g15_pb_max_blocks: u32 { default: 3072, - description: "G15: parameter-buffer growth maximum given to the firmware, blocks of 128 KiB (1024..4095 in effect; the blocks above it, up to 4095, answer partial renders; the 512 MiB heap is allocated either way)", + description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", + }, + g15_pb_blocks: u32 { + default: 64, + description: "G15: PB blocks posted at (re)initialization, 128 KiB each (64..4095)", }, g15_early_tiling: u32 { default: 0, diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index d566a54953e072..c7f21a9110cee8 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -571,11 +571,10 @@ pub(crate) mod tuning { pub(crate) const SLOT_ROTATE: bool = true; /// Flights per VM. pub(crate) const MAX_INFLIGHT: u32 = 2; - /// PB growth through event 7 / DEVCTL 7, initial and maximum blocks. + /// PB growth through event 7 / DEVCTL 7; the initial posting is `g15_pb_blocks` (64). pub(crate) const PB_GROW: bool = true; - /// Initial PB blocks, and the most that may be posted: one below the mapped heap's block - /// count, so the block-list producer (modulo K = 4096) never returns to 0. - pub(crate) const PB_BLOCKS: u32 = 1024; + /// The most PB blocks that may be posted: one below the mapped heap's block count, so the + /// block-list producer (modulo K = 4096) never returns to 0. pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS - 1; /// At the host deadline, arm the firmware kill. pub(crate) const HANG_PREEMPT: bool = true; diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index 6eabb033a0edf1..d771f8493ae26d 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -64,13 +64,19 @@ pub(crate) fn accept_text_prefix() -> u64 { } /// The parameter for the PB growth maximum the firmware is given, in blocks, clamped to -/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial 1024 blocks; -/// the blocks above it, up to the 4095 posted at most, answer the synchronous requests of -/// partial renders. It is read-only after boot and does not change the 512 MiB heap. +/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial posting; the +/// blocks above it, up to the 4095 posted at most, answer the synchronous requests of partial +/// renders. It is read-only after boot and does not change the heap's backing. pub(crate) fn pb_max_blocks() -> u32 { (*module_parameters::g15_pb_max_blocks.value()).clamp(64, super::hw::tuning::PB_MAX_BLOCKS) } +/// The initial PB posting, in blocks (64..=4095; the ledger clamps it again). Read-only after +/// boot. +pub(crate) fn pb_blocks() -> u32 { + (*module_parameters::g15_pb_blocks.value()).clamp(64, super::hw::tuning::PB_MAX_BLOCKS) +} + /// Whether FRAGMENT_BARRIERS is advertised and mapped to early tiling. pub(crate) fn early_tiling() -> bool { *module_parameters::g15_early_tiling.value() != 0 diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index 15204f60916138..50f8501e7dfcfc 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -12,9 +12,9 @@ //! the fixed client GVA: each VM's before its prefixes are published ([`Firmware::map_pb`]), //! and a chunk backed later also into the VM of the flights; nothing names its pages through //! the host window or context 0 (the records hold client GVAs only). Initial -//! [`super::hw::tuning::PB_BLOCKS`] (1024), at most [`super::hw::tuning::PB_MAX_BLOCKS`] -//! (4095) posted, so the producer never returns to 0; the firmware is given -//! [`Pools::soft_max`] (default 3072) as its growth maximum. Truthful K = 4096 in every +//! `g15_pb_blocks` (64), at most [`super::hw::tuning::PB_MAX_BLOCKS`] (4095) posted, so the +//! producer never returns to 0; the firmware is given [`Pools::soft_max`] (default 3072) as +//! its growth maximum. Truthful K = 4096 in every //! profile (PB reconstruction, also in recovery, synchronizes 8 * K bytes of the block list: //! K = 0x26cd overran the 0x8000-byte list and crashed the second recovery). //! - Backing: a request is granted from the backed blocks above the posted count; after the @@ -72,10 +72,10 @@ static_assert!(buffer::MAX_BLOCKS as u64 * buffer::BLOCK_BYTES == client::PB_HEA static_assert!(buffer::PAGE_LIST_SIZE as u64 == client::PB_PAGE_LIST_SIZE); static_assert!(uma_manager::SEEDS == sizes::FREELIST_SEED); static_assert!(uma_pools::POOL_SIZE == sizes::POOL_SIZE); -// The ledger keeps the driver's PB sizes: neither is capped by [`growth::MAX_POSTED`]. +// The ledger keeps the driver's PB maximum: it is not capped by [`growth::MAX_POSTED`]. The +// initial posting (`g15_pb_blocks`) is clamped to MIN_BLOCKS..=PB_MAX_BLOCKS. static_assert!(tuning::PB_MAX_BLOCKS <= growth::MAX_POSTED); -static_assert!(buffer::MIN_BLOCKS <= tuning::PB_BLOCKS); -static_assert!(tuning::PB_BLOCKS <= tuning::PB_MAX_BLOCKS); +static_assert!(buffer::MIN_BLOCKS <= tuning::PB_MAX_BLOCKS); // The PB alias is GPU-only (not cached and firmware-visible): dropping one is a plain unmap, // with no firmware cache flush (`KernelMapping::drop`). static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent()); @@ -373,7 +373,11 @@ impl Pools { true, )?; // Backing for the initial blocks and the reserve; growth adds the rest. - let mut ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let mut ledger = Ledger::new( + tuning::PB_GROW, + super::params::pb_blocks(), + tuning::PB_MAX_BLOCKS, + ); let mut chunks = KVec::with_capacity(growth::MAX_CHUNKS as usize, GFP_KERNEL)?; for _ in 0..Ledger::chunks_for(ledger.initial) { chunks.push(PbChunk::new(dev)?, GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 3ed143ee546ad5..b94c5c58be7337 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -257,11 +257,11 @@ mod tests { assert_eq!(Ledger::new(true, 512, 100).backed, 0); let fixed = Ledger::new(false, 512, 1024); assert_eq!((fixed.initial, fixed.max, fixed.posted), (4095, 4095, 4095)); - // The values the driver passes (tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS) are kept. - let l = Ledger::new(true, 1024, 4095); + // The values the driver passes (g15_pb_blocks 64, tuning::PB_MAX_BLOCKS) are kept. + let l = Ledger::new(true, 64, 4095); assert_eq!( (l.initial, l.max, l.posted, l.producer()), - (1024, 4095, 1024, 1024) + (64, 4095, 64, 64) ); } diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index aa321938476065..78d36f21075fb4 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -5,9 +5,9 @@ The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), answer_request(), map_pb() and refill_backing(), Pools::soft_max(), map_heap() and back_chunk(), PbChunk, chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from -vm.rs VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() from -params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and -is_cached_noncoherent() from pgtable.rs; and the parameter's default from asahi_core.rs. It +vm.rs VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() and +pb_blocks() from params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and +is_cached_noncoherent() from pgtable.rs; and the parameters' defaults from asahi_core.rs. It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() and the DEVCTL wait, which logs every allocation, alias map, unmap, free, block record, barrier, producer @@ -184,7 +184,7 @@ 'three-refusals': ('default_growth_to_exhaustion', 'want Some(Log("warn"'), 'never-exhausted': ('default_growth_to_exhaustion', 'growth did not end'), 'warn-every-time': ('default_growth_to_exhaustion', 'later request after exhaustion'), - 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(1034)), want Some(Sync)'), + 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(74)), want Some(Sync)'), 'no-reply-barrier': ('default_growth_to_exhaustion', 'got Some(Reply([7, 1, 0, 5, 0, 256, 3, 1, 0, 0, 0, 0, 0, 0])), want Some(Sync)'), 'soft-boundary-inclusive': ('soft_max_boundaries', 'got Some(Log("info", "")), want Some(Log("dbg", ""))'), @@ -334,6 +334,7 @@ def consts(text, names): thread_local! { pub static LOG: RefCell> = RefCell::new(Vec::new()); pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); + pub static PARAM_BLOCKS: Cell = Cell::new(PB_BLOCKS_DEFAULT); /// Failures to inject: the next this many allocations or alias maps fail. pub static FAIL_ALLOC: Cell = Cell::new(0); pub static FAIL_MAP: Cell = Cell::new(0); @@ -382,6 +383,13 @@ def consts(text, names): } } pub static g15_pb_max_blocks: Param = Param; + pub struct Blocks; + impl Blocks { + pub fn value(&self) -> &'static u32 { + Box::leak(Box::new(crate::PARAM_BLOCKS.with(|p| p.get()))) + } + } + pub static g15_pb_blocks: Blocks = Blocks; } pub mod driver { @@ -875,9 +883,10 @@ def consts(text, names): #[test] fn driver_values() { - let l = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let l = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); assert!(tuning::PB_GROW); - assert_eq!((l.initial, l.max, l.posted), (1024, 4095, 1024)); + assert_eq!(crate::PB_BLOCKS_DEFAULT, 64); + assert_eq!((l.initial, l.max, l.posted), (64, 4095, 64)); assert_eq!(PARAM_DEFAULT, 3072); PARAM.with(|p| p.set(PARAM_DEFAULT)); assert_eq!(params::pb_max_blocks(), 3072); @@ -890,8 +899,16 @@ def consts(text, names): for param in [0, 1, 63, 64, 65, 1023, 1024, 1025, 3071, 3072, 3073, 4094, 4095, 4096, 8192, u32::MAX] { PARAM.with(|p| p.set(param)); assert_eq!(params::pb_max_blocks(), clamp(param, 64, tuning::PB_MAX_BLOCKS), "pb_max_blocks, param {param}"); - // Below the initial 1024 the firmware still gets 1024. - assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}"); + // Below the initial 64 the firmware still gets 64. + assert_eq!(f.pools.soft_max(), clamp(param, 64, CAP), "soft_max, param {param}"); + } + // With a larger initial posting the firmware gets at least that. + crate::PARAM_BLOCKS.with(|b| b.set(1024)); + let f = Firmware::new(); + assert_eq!(f.pools.ledger.initial, 1024); + for param in [0, 64, 1023, 1024, 3072, 4095, u32::MAX] { + PARAM.with(|p| p.set(param)); + assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}, 1024 initial"); } } @@ -902,7 +919,7 @@ def consts(text, names): #[test] fn soft_max_boundaries() { - // 3074 is reached exactly (1024 + 205 * 10): that answer is not past it. + // 3074 is reached exactly (64 + 301 * 10): that answer is not past it. for param in [0, 1024, 3072, 3074, 3075, 4094, 4095, u32::MAX] { run_to_exhaustion(param); } @@ -959,7 +976,7 @@ def consts(text, names): assert_eq!(f.pools.stats.requests, 2, "requests taken in the pass"); assert_eq!(f.pools.stats.bad, 1); assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -5")))); - assert!(log.contains(&Ev::Producer(1024 + GROW_STEP))); + assert!(log.contains(&Ev::Producer(64 + GROW_STEP))); assert!(f.events.grow.is_empty()); } @@ -973,7 +990,7 @@ def consts(text, names): take(); f.pass(); let log = take(); - assert_eq!(f.pools.ledger.posted, 1024 + 3 * GROW_STEP); + assert_eq!(f.pools.ledger.posted, 64 + 3 * GROW_STEP); assert_eq!((f.pools.stats.grown, f.pools.stats.bad), (3 * GROW_STEP as u64, 0)); // Each failed reply is logged at info level. let info = log.iter().filter(|e| matches!(e, Ev::Log("info", m) if m.contains("DEVCTL 7: -5"))).count(); @@ -1006,7 +1023,7 @@ def consts(text, names): assert!(log.iter().all(|e| matches!(e, Ev::Log("warn", _))), "{log:?}"); assert_eq!(log.len(), 4); assert_eq!(f.pools.stats.bad, 4); - let mut fresh = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let mut fresh = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); fresh.backed = Ledger::chunks_for(fresh.initial) * CHUNK_BLOCKS; assert_eq!(f.pools.ledger, fresh); // No render published: not live. @@ -1089,7 +1106,7 @@ def consts(text, names): let log = take(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); let chunks = Ledger::chunks_for(initial) as usize; - assert_eq!(chunks, 9); + assert_eq!(chunks, 2); let want: Vec = (1..=chunks as u32).map(|id| Ev::Alloc(id, CHUNK_BYTES)).collect(); assert_eq!(log, want, "startup allocation"); assert_eq!((f.pools.ledger.backed, f.pools.chunk_count()), (chunks as u32 * CHUNK_BLOCKS, chunks)); @@ -1299,10 +1316,9 @@ def consts(text, names): f.queue(); f.pass(); let log = take(); - assert!(log.contains(&Ev::Producer(1034))); - // The grant crossed the reserve line: one chunk backed after it. - assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (1034, chunks + 1)); - assert_cursor(&f, &f.engine.flights[0].job.vm, chunks + 1); + assert!(log.contains(&Ev::Producer(74))); + assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (74, chunks)); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); } @@ -1310,10 +1326,10 @@ def consts(text, names): fn other_vms_catch_up_before_they_run() { let mut f = Firmware::new(); let a = f.engine.flights[0].job.vm.clone(); - grow_to(&mut f, 1100); + grow_to(&mut f, 200); take(); let chunks = f.pools.chunk_count(); - assert_eq!(chunks, 10); + assert_eq!(chunks, 3); // A's work retires; a new VM runs next: it maps every chunk first. f.engine.flights.clear(); let b = VmState::new(6); @@ -1325,7 +1341,7 @@ def consts(text, names): assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 0); assert_cursor(&f, &b, chunks); // B grows the backing; A, behind, catches up when it runs again. - grow_to(&mut f, 1200); + grow_to(&mut f, 300); let log = take(); assert_posted_backed(&log, 6, chunks as u64 * CHUNK_BLOCKS as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 0); @@ -1339,7 +1355,7 @@ def consts(text, names): #[test] fn partial_catch_up_keeps_the_aliases_made() { let mut f = Firmware::new(); - grow_to(&mut f, 1100); + grow_to(&mut f, 200); let chunks = f.pools.chunk_count(); f.engine.flights.clear(); let b = VmState::new(6); @@ -1358,12 +1374,12 @@ def consts(text, names): #[test] fn ledger_reset_reuses_the_backing() { let mut f = Firmware::new(); - grow_to(&mut f, 1300); + grow_to(&mut f, 300); let chunks = f.pools.chunk_count(); take(); f.reset_ledger(); - assert_eq!(f.pools.ledger.posted, 1024); - grow_to(&mut f, 1300); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 300); let log = take(); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..) | Ev::Map(..))), 0); assert_eq!(f.pools.chunk_count(), chunks); @@ -1421,7 +1437,7 @@ def consts(text, names): // Only inputs the old constructor took to the whole 4096-block heap changed. assert!(changed > 0 && changed < cases); // The driver's values are below the cap, so its ledger is the old one. - let d = (tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let d = (tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); let (old, new) = (OldLedger::new(d.0, d.1, d.2), Ledger::new(d.0, d.1, d.2)); if d.2 <= CAP { assert_eq!((old.initial, old.max, old.posted), (new.initial, new.max, new.posted)); @@ -1507,13 +1523,18 @@ def run(mutation): if init.count(line) != 1: raise SystemExit('Pools::init() does not hold: ' + line) reset = '\n'.join('p.' + line[len('self.'):] for line in LEDGER_RESET) - pb_max_blocks = item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> u32 {') + pb_max_blocks = (item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> u32 {') + '\n' + + item(src[PARAMS], 'pub(crate) fn pb_blocks() -> u32 {')) m = re.search(r'g15_pb_max_blocks: u32 \{\s*default: (\d+),', src[CORE]) if not m: raise SystemExit('g15_pb_max_blocks default not found') param_default = int(m.group(1)) + m = re.search(r'g15_pb_blocks: u32 \{\s*default: (\d+),', src[CORE]) + if not m: + raise SystemExit('g15_pb_blocks default not found') + blocks_default = int(m.group(1)) sizes = consts(item(hw, 'pub(crate) mod sizes {'), ['PB_BLOCKS']) - tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_BLOCKS', 'PB_MAX_BLOCKS']) + tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_MAX_BLOCKS']) client = consts(item(hw, 'pub(crate) mod client {'), ['PB_HEAP', 'PB_BLOCK_LIST']) pte = consts(item(hw, 'pub(crate) mod pte {'), ['LEAF_CACHED']) control = consts(item(src[INITDATA], 'pub(crate) mod control {'), ['HALT_COUNT']) @@ -1527,6 +1548,7 @@ def run(mutation): for path in (GROWTH, BUFFER, CHANNELS): (dir / Path(path).name).write_text(src[path]) code = (PREFIX + f'pub const PARAM_DEFAULT: u32 = {param_default};\n' + + f'pub const PB_BLOCKS_DEFAULT: u32 = {blocks_default};\n' + 'pub mod g15 {\n' ' pub mod fw {\n' f' #[path = "{dir / "channels.rs"}"] pub mod channels;\n' diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index 5b728e95fc03e2..e659a507477b56 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -5,8 +5,10 @@ Uses synthetic public data and temporary host copies of production Rust modules. No GPU, firmware file, private fixture, kernel build or hardware access. The golden T6030 inputs below are from SiliconForest 9f09c85f8d3b, except the full -hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes. -Its T6030 configuration module is independently pinned to the original bytes. +hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes +and again when the initial PB posting became a module parameter (the tuning +constant PB_BLOCKS removed). Its T6030 configuration module is independently +pinned to the original bytes. """ import argparse @@ -22,7 +24,7 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "e4619590e670ed88d2a9bb76b03853081c615d3d0ad1c6a68321ce8e615ef8a0", + "hw.rs": "658c58026553c098b899aeffa0b6ae5be2b14921aef19df9e4f14789b7b890c4", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { From 54941febebcff7705d50698e24d244afb10034be Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:56 +0530 Subject: [PATCH 06/11] drm/asahi: g15: m3: reclaim the parameter buffer when the engine is idle Posted blocks never come back while a PB lives. The firmware recycles pages inside its own pool and the records give the host no view of it, so the posted count only grows until a recovery re-initializes the PB. On the J516C one dEQP case takes about 1200 blocks past the firmware's maximum each time it runs. The second run of one dEQP list exhausted the 512 MiB heap, and the GPU fail-stopped after the refusals. Re-initialize the PB when the engine has had nothing published and nothing pending for g15_pb_idle_ms (1000), with more than g15_pb_reclaim_blocks (1024) posted: - the next render prefix takes the re-initialization path of a recovered abort (fresh lists and manager, InitBM of the initial blocks), with no halt; - with g15_pb_reclaim 2 (the default), the chunks above those of the initial posting and the reserve are freed at once and unmapped from every VM of the registry; a VM that publishes later drops its stale aliases first (VmState::map_heap()); a chunk counts as freed only once every VM's alias of it is unmapped; - g15_pb_reclaim 1 re-initializes only, 0 never; - nothing is re-initialized when nothing grew since the initialization (posted == initial), whatever g15_pb_reclaim_blocks is. The idle clock is noted at the end of every poll pass and publication and when a job arrives. The check runs in idle poll passes and when a job arrives at an idle engine, before its publication, because an idle poll is not re-armed. This applies to the T6030 and the T6031, which share this code; it is untested on the T6030 (g15_pb_reclaim=0 turns it off, and g15_pb_blocks=1024 restores the old start). The debugfs pb: line appends reclaims and freed chunks. Host tests: the idle clock, the reclaim test of the ledger, a heavy scene freed and refilled, and (growth harness) a VM that maps chunks freed and backed again from their new objects. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/asahi/asahi_core.rs | 12 ++ drivers/gpu/drm/asahi/g15/engine.rs | 7 ++ drivers/gpu/drm/asahi/g15/params.rs | 15 +++ drivers/gpu/drm/asahi/g15/poll.rs | 6 + drivers/gpu/drm/asahi/g15/pools.rs | 116 ++++++++++++++++++- drivers/gpu/drm/asahi/g15/pools/growth.rs | 134 +++++++++++++++++++++- drivers/gpu/drm/asahi/g15/vm.rs | 11 ++ tools/asahi/g15-pb-growth-tests.py | 48 +++++++- 8 files changed, 342 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 084718c5b1dcbe..a55322d50ad236 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -143,6 +143,18 @@ module! { default: 64, description: "G15: PB blocks posted at (re)initialization, 128 KiB each (64..4095)", }, + g15_pb_reclaim: u32 { + default: 2, + description: "G15: PB reclaim when idle: 0 never, 1 re-initialize, 2 also free backing", + }, + g15_pb_idle_ms: u32 { + default: 1000, + description: "G15: idle time before a PB reclaim, ms (at least 100)", + }, + g15_pb_reclaim_blocks: u32 { + default: 1024, + description: "G15: PB blocks posted above which an idle engine reclaims the PB", + }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index e5c18a15e0c3b3..33974222afe9b0 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -515,8 +515,12 @@ impl Firmware { if self.engine.flights.capacity() < sizes::FLIGHTS { self.engine.flights.reserve(sizes::FLIGHTS, GFP_KERNEL)?; } + // A job arriving at an idle engine: the PB reclaim, if due, happens before its + // publication (idle polls are not re-armed, so this is where a long idle ends). + self.reclaim_check(); self.engine.pending.push(job.clone(), GFP_KERNEL)?; job.progress.set_pending(true); + self.note_idle(); let q = &job.queue; let owned = self.engine.queue_published(&job) + self @@ -601,6 +605,9 @@ impl Firmware { } } } + // A publication that left nothing published and nothing pending (jobs that failed + // before any visible record) starts the idle. + self.note_idle(); result } diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index d771f8493ae26d..48d21e45d0acd6 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -77,6 +77,21 @@ pub(crate) fn pb_blocks() -> u32 { (*module_parameters::g15_pb_blocks.value()).clamp(64, super::hw::tuning::PB_MAX_BLOCKS) } +/// PB reclaim at idle: 0 never, 1 re-initialize, 2 re-initialize and free the backing. +pub(crate) fn pb_reclaim() -> u32 { + (*module_parameters::g15_pb_reclaim.value()).min(2) +} + +/// Idle time before a PB reclaim, ns (at least 100 ms). +pub(crate) fn pb_idle_ns() -> u64 { + (*module_parameters::g15_pb_idle_ms.value()).max(100) as u64 * 1_000_000 +} + +/// Posted blocks above which an idle engine reclaims the PB. +pub(crate) fn pb_reclaim_blocks() -> u32 { + *module_parameters::g15_pb_reclaim_blocks.value() +} + /// Whether FRAGMENT_BARRIERS is advertised and mapped to early tiling. pub(crate) fn early_tiling() -> bool { *module_parameters::g15_early_tiling.value() != 0 diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 751f19865436f8..36f27b3aa58568 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -207,6 +207,9 @@ impl Firmware { // prefixes published the retirement loop below drains first. if self.engine.flights.is_empty() { self.drain_rings()?; + // Idle bookkeeping of the PB: when the idle began, and the reclaim once it is due. + self.note_idle(); + self.reclaim_check(); if self.recovery_idle() { return Ok(false); } @@ -228,6 +231,9 @@ impl Firmware { self.engine.prof.polls += 1; self.engine.prof.poll_ns += now_ns().wrapping_sub(start); } + // The pass that retired the last flight is not re-armed: note the idle here, not only + // at the start of a pass that finds the flights empty. + self.note_idle(); published?; Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen) } diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index 50f8501e7dfcfc..f9660a616c5d43 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -21,6 +21,12 @@ //! replies of a pass, while fewer than a reserve chunk remain, [`Firmware::refill_backing`] //! backs the next chunk, so a reply never waits for the allocator. Only a reserve that ran dry //! makes a request wait: it stays queued, the pass backs a chunk and the next pass answers it. +//! - Reclaim ([`Firmware::reclaim_check`], `g15_pb_reclaim`): posted blocks never come back +//! while a PB lives (the firmware recycles pages inside its own pool; the host has no view of +//! it), so an engine idle for `g15_pb_idle_ms` with more than `g15_pb_reclaim_blocks` posted +//! re-initializes the PB at its next render through the re-initialization path of a recovered +//! abort (fresh lists and manager, InitBM of the initial blocks, no halt) and frees the backing +//! above the kept chunks. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -62,7 +68,7 @@ use super::Firmware; use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{gem, mmu}; -pub(crate) use growth::{Answer, Ledger}; +pub(crate) use growth::{Answer, IdleClock, Ledger}; static_assert!(buffer::MAX_BLOCKS == sizes::PB_BLOCKS); static_assert!(buffer::MAX_PAGES == sizes::PB_PAGES); @@ -158,6 +164,9 @@ pub(crate) struct PbStats { pub(crate) waits: u64, /// Chunks backed after the GPU started. pub(crate) refills: u64, + /// Idle re-initializations, and the chunks they freed. + pub(crate) reclaims: u64, + pub(crate) freed: u64, } /// Render allocator and PB state: the allocators, the PB ledger and the spare set. @@ -185,6 +194,8 @@ pub(crate) struct Pools { c1_written: bool, /// The unanswered-at-the-maximum warning was given since the last initialization. exhausted_logged: bool, + /// The engine's idle clock (nothing published, nothing pending), for the reclaim. + idle: IdleClock, pub(crate) stats: PbStats, } @@ -403,6 +414,7 @@ impl Pools { ledger, c1_written: false, exhausted_logged: false, + idle: IdleClock::default(), stats: PbStats::default(), }) } @@ -539,6 +551,15 @@ impl Pools { vm.map_heap(self.chunks.iter().map(PbChunk::gem)) } + /// Frees the backing chunks from index `keep` on (their VM mappings are dropped by the + /// caller); the RAM returns when the last mapping of a chunk is gone. + fn free_chunks(&mut self, keep: usize) -> usize { + let freed = self.chunks.len().saturating_sub(keep); + self.chunks.truncate(keep); + self.ledger.unback_to(self.chunks.len() as u32); + freed + } + /// Backs one more chunk; the caller maps it where needed before the ledger posts from it. fn back_chunk(&mut self) -> Result> { if self.chunks.len() >= growth::MAX_CHUNKS as usize { @@ -551,11 +572,11 @@ impl Pools { } /// The debugfs line `pb: blocks initial/posted/firmware-max/max backed blocks/chunks - /// requests grown refused bad waits refills` (new keys are appended). + /// requests grown refused bad waits refills reclaims freed` (new keys are appended). pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { writeln!( f, - "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {} waits {} refills {}", + "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {} waits {} refills {} reclaims {} freed {}", self.ledger.initial, self.ledger.posted, self.soft_max(), @@ -567,7 +588,9 @@ impl Pools { self.stats.refused, self.stats.bad, self.stats.waits, - self.stats.refills + self.stats.refills, + self.stats.reclaims, + self.stats.freed ) } } @@ -660,6 +683,91 @@ impl Firmware { } } + /// Notes the engine idle (nothing published, nothing pending) or busy; the idle time is + /// what [`Self::reclaim_check`] measures. Called at the end of every poll pass and + /// publication (the pass that retires the last flight is not re-armed) and at a job's + /// arrival. + pub(crate) fn note_idle(&mut self) { + let idle = self.engine.flights.is_empty() && self.engine.pending.is_empty(); + self.pools.idle.note(idle, super::engine::now_ns()); + } + + /// Reclaims the PB when the engine has been idle for `g15_pb_idle_ms` with more than + /// `g15_pb_reclaim_blocks` posted: the next render prefix re-initializes the render + /// allocators and the PB (InitBM of the initial blocks; `recovery.reinit`, as after a + /// recovered abort but with no halt), and with `g15_pb_reclaim` 2 the backing chunks above + /// those of the initial posting and the reserve are freed now, unmapped from every VM of the + /// registry (a VM that publishes later drops its stale mappings at [`VmState::map_heap`]). + /// Called under the device lock when a job arrives at an idle engine and in idle poll + /// passes. + pub(crate) fn reclaim_check(&mut self) { + let level = super::params::pb_reclaim(); + if level == 0 + || !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || self.recovery.reinit + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + let floor = super::params::pb_reclaim_blocks(); + if !self + .pools + .ledger + .reclaim_due(floor, idle_ns, super::params::pb_idle_ns()) + { + return; + } + self.recovery.reinit = true; + self.pools.stats.reclaims += 1; + let posted = self.pools.ledger.posted; + let mut freed = 0; + if level >= 2 { + freed = self.free_backing(); + } + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted: re-initialization to {} at the next render, {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + self.pools.ledger.initial, + freed, + self.pools.chunks.len() + ); + } + + /// Frees the backing chunks above those a re-initialized heap keeps + /// ([`growth::Ledger::keep_chunks`]) and drops their mappings in every VM that could + /// publish; returns the chunks freed. If the VMs cannot be walked, their mappings keep the + /// pages until each VM drops its stale aliases ([`VmState::map_heap`]): nothing is counted + /// freed. + fn free_backing(&mut self) -> usize { + let keep = self.pools.ledger.keep_chunks() as usize; + let dropped = self.pools.free_chunks(keep); + // The mappings hold the chunks' pages: drop them in every VM that could publish. + let freed = match self.vms.open() { + Ok(states) => { + for state in states.iter() { + state.unmap_heap_above(keep); + } + dropped + } + Err(e) => { + dev_warn!( + self.dev(), + "PB heap: reclaim could not walk the VMs: {}\n", + e.to_errno() + ); + 0 + } + }; + self.pools.stats.freed += freed as u64; + freed + } + /// (Re)initializes the render UMA managers, PB lists and PB manager. `rebuild`: the priming /// prefix after a recovered abort: the first one moves to the uncached spare set, later /// ones rewrite it in place. diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index b94c5c58be7337..1790699e2d7ac4 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -18,7 +18,8 @@ //! whenever fewer than [`RESERVE_BLOCKS`] backed blocks remain above the posted count, so that //! it can be backed after a reply; a request that finds no backed block waits //! ([`Answer::Wait`]) and keeps its counter. A re-initialization ([`Ledger::reinit`]) returns -//! the posted count to the initial size and keeps the backing. +//! the posted count to the initial size and keeps the backing; [`Ledger::keep_chunks`] gives the +//! chunks that the initial posting and the reserve need. //! //! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. @@ -121,6 +122,11 @@ impl Ledger { self.backed += CHUNK_BLOCKS; } + /// Drops the backing to `chunks` chunks. + pub(crate) const fn unback_to(&mut self, chunks: u32) { + self.backed = chunks * CHUNK_BLOCKS; + } + /// Backed blocks above the posted count. pub(crate) const fn spare(&self) -> u32 { self.backed.saturating_sub(self.posted) @@ -142,6 +148,18 @@ impl Ledger { } } + /// Chunks a re-initialized heap keeps: those of the initial posting and the reserve. + pub(crate) const fn keep_chunks(&self) -> u32 { + Self::chunks_for(self.initial) + } + + /// Whether the heap is to be re-initialized: idle for `idle_ns` with more than `floor` + /// blocks posted, and grown since the initialization (with nothing grown a + /// re-initialization posts the same blocks again). + pub(crate) const fn reclaim_due(&self, floor: u32, idle_ns: u64, min_idle_ns: u64) -> bool { + self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -203,6 +221,37 @@ impl Ledger { } } +/// The engine's idle clock for the reclaim policy: when nothing was published and nothing +/// pending, from the pass that retired the last flight on. Noted at the end of every poll pass +/// and publication (a pass that empties the flights is not re-armed, so a note only at the +/// start of a pass with empty flights depends on a backup pass happening to run) and, busy, +/// when a job is enqueued. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct IdleClock { + /// When the idle began; 0 while busy. + since_ns: u64, +} + +impl IdleClock { + /// Notes the engine idle or busy at `now_ns`; an idle already begun keeps its start. + pub(crate) fn note(&mut self, idle: bool, now_ns: u64) { + if !idle { + self.since_ns = 0; + } else if self.since_ns == 0 { + self.since_ns = now_ns.max(1); + } + } + + /// The idle time at `now_ns`; 0 while busy. + pub(crate) fn idle_ns(&self, now_ns: u64) -> u64 { + if self.since_ns == 0 { + 0 + } else { + now_ns.wrapping_sub(self.since_ns) + } + } +} + /// The halt epoch a request names (u64 at words 5..6). pub(crate) const fn request_epoch(request: &[u32; 14]) -> u64 { request[grow_request::EPOCH] as u64 | (request[grow_request::EPOCH + 1] as u64) << 32 @@ -401,6 +450,7 @@ mod tests { assert_eq!(Ledger::chunks_for(4095), MAX_CHUNKS); assert_eq!(MAX_CHUNKS, 32); let mut l = Ledger::new(true, 64, 4095); + assert_eq!(l.keep_chunks(), 2); assert_eq!(l.spare(), 0); assert!(l.refill_due()); // Nothing backed: a request waits and the counter does not advance. @@ -449,9 +499,11 @@ mod tests { ); l.post(5); assert_eq!(l.accept(&request(3, 3, 1)), Answer::Refuse); - // Re-initialization keeps the backing. + // Re-initialization keeps the backing; dropping it to the kept chunks. l.reinit(); assert_eq!((l.posted, l.backed), (64, 4096)); + l.unback_to(l.keep_chunks()); + assert_eq!(l.backed, 256); } /// Every ledger the constructor makes, the driver's included, backed a chunk at a time @@ -492,6 +544,84 @@ mod tests { } } + #[test] + fn reclaim() { + let mut l = Ledger::new(true, 64, 4095); + let s = 1_000_000_000; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1024; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1025; + assert!(l.reclaim_due(1024, s, s)); + assert!(!l.reclaim_due(1024, s - 1, s)); + // The clock: busy is 0, an idle keeps its start, busy ends it. + let mut c = IdleClock::default(); + assert_eq!(c.idle_ns(10 * s), 0); + c.note(true, 10 * s); + c.note(true, 12 * s); + assert_eq!(c.idle_ns(15 * s), 5 * s); + c.note(false, 16 * s); + assert_eq!(c.idle_ns(20 * s), 0); + // A clock started at 0 ns still counts as idle. + c.note(true, 0); + assert_eq!(c.idle_ns(s), s - 1); + // Nothing grown since the initialization: no re-initialization, whatever the floor. + let l = Ledger::new(true, 64, 4095); + assert!( + !l.reclaim_due(32, 5 * s, s), + "re-initialization with nothing grown" + ); + } + + /// A heavy scene, a reclaim that frees the backing, refills during the next heavy scene, + /// an idle gap with a small job in it, and the next render must reclaim again. + #[test] + fn reclaim_after_free_then_refill() { + let s = 1_000_000_000; + let (floor, min_idle) = (1024, s); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + let mut clock = IdleClock::default(); + // A heavy scene grows the PB to 1484 blocks, refilling as it goes. + let heavy = |l: &mut Ledger, clock: &mut IdleClock, t: u64| { + clock.note(false, t); + let mut counter = l.next_request; + while l.posted < 1484 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + // The pass that retires the last flight notes the idle; no later pass may run. + clock.note(true, t + s); + }; + heavy(&mut l, &mut clock, 100 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // 5 s later a small job arrives: the reclaim is due; it frees the backing above the + // kept chunks; the re-initialization happens at the next render. + let t = 106 * s; + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + l.unback_to(l.keep_chunks()); + clock.note(false, t); + assert_eq!(l.backed, 2 * CHUNK_BLOCKS); + clock.note(true, t + s / 2); + // The next render re-initializes, then the second heavy scene refills to 1484 again. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 256)); + heavy(&mut l, &mut clock, 110 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // The same gap again: the reclaim must be due at the small job's arrival. + let t = 116 * s; + assert_eq!(clock.idle_ns(t), 5 * s); + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + } + #[test] fn grants_only_backed_blocks() { let mut l = Ledger::new(true, 64, 4095); diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index e8e49adea03110..e2b9d884bc25b8 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -321,6 +321,17 @@ impl Registry { .ok_or(ENOENT) } + /// The VMs whose client handle is open (and the closed ones not pruned yet), for a walk + /// that takes VM locks: not under the registry lock. + pub(crate) fn open(&self) -> Result>> { + let vms = self.vms.lock(); + let mut out = KVec::with_capacity(vms.len(), GFP_KERNEL)?; + for state in vms.iter() { + out.push(state.clone(), GFP_KERNEL)?; + } + Ok(out) + } + /// Latches `error` on every VM whose client handle is open. The fail-stop path calls it under /// the device lock. pub(crate) fn fail_all(&self, error: Error) { diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 78d36f21075fb4..5c3c2ac7452d32 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -37,7 +37,8 @@ render is published, and still backs for a render published before it; a failed reply keeps everything; - a VM's aliases are the chunks, contiguous from the heap's start; a VM that runs after another - grew the backing maps it first (at publication); a partial catch-up keeps the aliases made; + grew the backing maps it first (at publication); a partial catch-up keeps the aliases made; a + chunk freed and backed again (an idle reclaim) is unmapped and mapped again at publication; - the one-VM rule: a catch-up with a flight of another VM is refused (EPROTO) before anything is mapped; - the ledger reset of Pools::init() re-posts from the backing without allocating; @@ -164,6 +165,8 @@ # Refill for any live flight, also compute-only ones that cannot ask for growth. 'refill-for-any-flight': (POOLS, 'if self.recovery.reinit && !render {', 'if self.recovery.reinit && self.engine.flights.is_empty() {'), + # Keep a VM's alias of a chunk that was freed and backed again. + 'map-without-identity': (VM, ' if core::ptr::eq(&*have.gem, &*gem) {', ' if true {'), # Catch up without the one-VM check. 'no-activation-guard': (POOLS, '.filter(|f| !core::ptr::eq(&*f.job.vm, vm))', '.filter(|_| false)'), # Back the whole heap when the GPU starts. @@ -198,6 +201,7 @@ 'refill-during-reinit': ('pending_reinit_without_flights_backs_nothing', 'backed during a pending re-initialization'), 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), + 'map-without-identity': ('freed_chunks_are_mapped_again', 'alias of another object'), 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), @@ -576,6 +580,12 @@ def consts(text, names): pub fn chunk_capacity(&self) -> usize { self.chunks.capacity() } + /// What a reclaim does to the backing (the PB re-initialization included). + pub fn drop_chunks(&mut self, keep: usize) { + self.chunks.truncate(keep); + self.ledger.reinit(); + self.ledger.unback_to(keep as u32); + } /// The backing objects' ids, in heap order. pub fn chunk_ids(&self) -> Vec { self.chunks.iter().map(|c| c.backing.gem.id).collect() @@ -1398,6 +1408,42 @@ def consts(text, names): assert_eq!(f.pools.stats.bad, 1); } + /// What a reclaim leaves behind in a VM that was not walked (a closed VM not pruned yet, or + /// one created meanwhile): its aliases of freed chunks go, and chunks backed again at the + /// same GVA are mapped from their new objects at its next publication. + #[test] + fn freed_chunks_are_mapped_again() { + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 300); + let chunks = f.pools.chunk_count(); + assert!(chunks > 2); + // The reclaim frees the chunks above the kept two; A is not walked. + f.engine.flights.clear(); + f.pools.drop_chunks(2); + take(); + // B runs and grows the backing again: new objects at the old GVAs. + let b = VmState::new(6); + f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); + f.fw.ctx = 6; + f.map_pb(&b).unwrap(); + grow_to(&mut f, 300); + assert_eq!(f.pools.chunk_count(), chunks); + // A runs again: its stale aliases are replaced by the new objects'. + f.engine.flights.clear(); + f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); + take(); + f.map_pb(&a).unwrap(); + let log = take(); + assert_cursor(&f, &a, chunks); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), chunks - 2); + // With fewer chunks than A maps, the ones past the backing go too. + f.pools.drop_chunks(2); + f.map_pb(&a).unwrap(); + assert_cursor(&f, &a, 2); + } + #[test] fn catch_up_refuses_flights_of_another_vm() { let mut f = Firmware::new(); From 59aced969cbefdc9ac2801276c15e91c97de6065 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:57 +0530 Subject: [PATCH 07/11] drm/asahi: g15: m3: re-initialize the parameter buffer at a client boundary The idle reclaim needs g15_pb_idle_ms without work. Heavy scenes started back to back, each by a new client a few hundred ms after the last, never reach it. On the J516C three such scenes of about 1420 blocks each ran the heap to its maximum, and the fourth hung the GPU into fail-stop. Re-initialize also after 50 ms idle with more than half the maximum posted and grown since the initialization (an initial posting above half the maximum is not drained). 50 ms is longer than the gaps between one client's renders. On the J516C a re-initialization with no idle time faulted at the heap base, at a client switch, the second time in a row with no recovery between; with 50 ms of idle none faulted, also between two renders of one client 60 ms apart. This path keeps the backing, so the next scene does not allocate it again. The idle reclaim now also runs when more chunks are backed than the posted blocks need (free_due()), so backing kept this way is freed even if the next client posts below g15_pb_reclaim_blocks. With no render after the client boundary, the re-initialization stays pending, and with g15_pb_reclaim 2 the check frees the backing above the kept chunks once the engine has been idle for g15_pb_idle_ms (a recovery's pending re-initialization still stops the check). This applies to the T6030 and the T6031, which share this code; it is untested on the T6030 (g15_pb_reclaim=0 turns it off, and g15_pb_blocks=1024 restores the old start). Host tests: back-to-back heavy scenes never reach the maximum and keep their backing, and kept backing is freed at the next long idle. The growth harness also runs reclaim_check(), free_backing() and note_idle() over a clock: a heavy client that exits with nothing rendered after it is re-initialized after 50 ms and its kept backing freed after the long idle, its posted count kept until the InitBM; at level 1 the backing stays; a recovery's pending re-initialization frees nothing; nothing grown, nothing is re-initialized. The reclaim keeps the posted count until the next render's InitBM, as the driver does: the same client rendering again after 60 ms grows again from the kept backing with nothing allocated, after 1.1 s over freshly backed chunks mapped before they are posted, and a compute-only submission between the reclaim and the InitBM maps, backs and posts nothing. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/asahi/g15/pools.rs | 60 ++- drivers/gpu/drm/asahi/g15/pools/growth.rs | 115 ++++++ tools/asahi/g15-pb-growth-tests.py | 470 ++++++++++++++++++++-- 3 files changed, 597 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index f9660a616c5d43..f03b196760b6ac 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -26,7 +26,10 @@ //! it), so an engine idle for `g15_pb_idle_ms` with more than `g15_pb_reclaim_blocks` posted //! re-initializes the PB at its next render through the re-initialization path of a recovered //! abort (fresh lists and manager, InitBM of the initial blocks, no halt) and frees the backing -//! above the kept chunks. +//! above the kept chunks (also with fewer posted, when more is backed than they need); an +//! engine idle for 50 ms (a client boundary) with more than half the maximum posted +//! re-initializes it the same way but keeps the backing; if no render follows, with +//! `g15_pb_reclaim` 2 the backing above the kept chunks is freed after `g15_pb_idle_ms`. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -196,6 +199,9 @@ pub(crate) struct Pools { exhausted_logged: bool, /// The engine's idle clock (nothing published, nothing pending), for the reclaim. idle: IdleClock, + /// The pending re-initialization was set by a reclaim ([`Firmware::reclaim_check`]), not by + /// a recovery; cleared by [`Self::init`]. + reclaim_pending: bool, pub(crate) stats: PbStats, } @@ -415,6 +421,7 @@ impl Pools { c1_written: false, exhausted_logged: false, idle: IdleClock::default(), + reclaim_pending: false, stats: PbStats::default(), }) } @@ -476,6 +483,7 @@ impl Pools { // capacity. self.ledger.reinit(); self.exhausted_logged = false; + self.reclaim_pending = false; if tuning::PB_GROW { // C1 names the initial pages again. self.c1_written = false; @@ -693,19 +701,24 @@ impl Firmware { } /// Reclaims the PB when the engine has been idle for `g15_pb_idle_ms` with more than - /// `g15_pb_reclaim_blocks` posted: the next render prefix re-initializes the render - /// allocators and the PB (InitBM of the initial blocks; `recovery.reinit`, as after a - /// recovered abort but with no halt), and with `g15_pb_reclaim` 2 the backing chunks above - /// those of the initial posting and the reserve are freed now, unmapped from every VM of the - /// registry (a VM that publishes later drops its stale mappings at [`VmState::map_heap`]). - /// Called under the device lock when a job arrives at an idle engine and in idle poll - /// passes. + /// `g15_pb_reclaim_blocks` posted or more backing than the posted blocks need, or for + /// [`growth::DRAIN_IDLE_NS`] with more than half the maximum posted + /// ([`growth::Ledger::drain_reinit_due`]; re-initialization only): the next render prefix + /// re-initializes the render allocators and the PB (InitBM of the initial blocks; + /// `recovery.reinit`, as after a recovered abort but with no halt), and with + /// `g15_pb_reclaim` 2 the backing chunks above those of the initial posting and the reserve + /// are freed now, unmapped from every VM of the registry (a VM that publishes later drops + /// its stale mappings at [`VmState::map_heap`]). While a reclaim's re-initialization is + /// pending (a client-boundary one kept the backing), `g15_pb_reclaim` 2 frees the backing + /// above those chunks once the engine has been idle for `g15_pb_idle_ms` + /// ([`growth::Ledger::pending_free_due`]). Called under the device lock when a job arrives + /// at an idle engine and in idle poll passes. pub(crate) fn reclaim_check(&mut self) { let level = super::params::pb_reclaim(); if level == 0 || !self.engine.flights.is_empty() || !self.engine.pending.is_empty() - || self.recovery.reinit + || (self.recovery.reinit && !self.pools.reclaim_pending) || self.recovery.armed.is_some() || self.engine.frozen || self.recovery.failed().is_some() @@ -714,18 +727,35 @@ impl Firmware { } let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); let floor = super::params::pb_reclaim_blocks(); - if !self - .pools - .ledger - .reclaim_due(floor, idle_ns, super::params::pb_idle_ns()) - { + let min_idle = super::params::pb_idle_ns(); + let ledger = &self.pools.ledger; + if self.recovery.reinit { + if level >= 2 && ledger.pending_free_due(idle_ns, min_idle) { + let posted = ledger.posted; + let freed = self.free_backing(); + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted and the re-initialization pending: {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + freed, + self.pools.chunks.len() + ); + } + return; + } + let idle = ledger.reclaim_due(floor, idle_ns, min_idle) + || (level >= 2 && ledger.free_due(idle_ns, min_idle)); + if !idle && !ledger.drain_reinit_due(idle_ns) { return; } self.recovery.reinit = true; + self.pools.reclaim_pending = true; self.pools.stats.reclaims += 1; let posted = self.pools.ledger.posted; let mut freed = 0; - if level >= 2 { + // A short idle keeps the backing: the next scene would allocate it again. + if level >= 2 && idle { freed = self.free_backing(); } dev_info!( diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 1790699e2d7ac4..c98aef060f94e8 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -62,6 +62,10 @@ pub(crate) const RESERVE_BLOCKS: u32 = CHUNK_BLOCKS; /// Chunks of the whole heap. pub(crate) const MAX_CHUNKS: u32 = MAX_BLOCKS / CHUNK_BLOCKS; +/// The idle time before a re-initialization past half the maximum (a client boundary, not a +/// gap between one client's renders). +pub(crate) const DRAIN_IDLE_NS: u64 = 50_000_000; + /// The answer to an admitted growth request. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum Answer { @@ -160,6 +164,31 @@ impl Ledger { self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns } + /// Whether the backing is to be freed: idle for `idle_ns` with more chunks backed than the + /// posted blocks and the reserve need. A client-boundary re-initialization keeps the + /// backing of a heavy scene; if the next client posts below the reclaim floor, this frees it. + pub(crate) const fn free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS + } + + /// While a reclaim's re-initialization is pending (the posted count returns to the initial + /// size at the next render): whether the backing above [`Self::keep_chunks`] is to be + /// freed, idle for `min_idle_ns`. A client-boundary re-initialization keeps the backing; + /// with no render after it, this frees it. + pub(crate) const fn pending_free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > self.keep_chunks() * CHUNK_BLOCKS + } + + /// Whether a short idle re-initializes the heap: more than half the maximum posted and + /// idle for [`DRAIN_IDLE_NS`]. Back-to-back heavy scenes (a new client every few hundred + /// ms) never reach `g15_pb_idle_ms`, and posted blocks never come back while a PB lives, so + /// without this they run the heap to the maximum. A re-initialization with no idle time + /// faulted at the heap base (at a client switch, the second in a row with no recovery + /// between); with 50 ms of idle none did, also between two renders of one client. + pub(crate) const fn drain_reinit_due(&self, idle_ns: u64) -> bool { + self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -573,6 +602,77 @@ mod tests { ); } + /// Heavy scenes of ~1420 blocks each, new clients back to + /// back. The engine re-initializes at a client boundary once past half the maximum, so the + /// heap never reaches it; the backing stays (no refill on the next scene). + #[test] + fn back_to_back_heavy_scenes() { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + let mut counter = 0; + for _ in 0..10 { + if l.drain_reinit_due(DRAIN_IDLE_NS) { + l.reinit(); + counter = l.next_request; + } + let start = l.posted; + while l.posted < start + 1420 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + assert!(l.posted < l.max); + } + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + assert_eq!(l.backed, 24 * CHUNK_BLOCKS); + l.posted = 2047; + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + l.posted = 2048; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // Not between two renders of one client. + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS - 1)); + assert!(!l.drain_reinit_due(0)); + // An initial posting above half the maximum: nothing grew, nothing to drain. + let l = Ledger::new(true, 3000, 4095); + assert!( + !l.drain_reinit_due(DRAIN_IDLE_NS), + "drain with nothing grown" + ); + } + + /// A client-boundary re-initialization keeps the backing + /// of a heavy scene, the next client posts little; once idle, the backing above what the + /// posted blocks need is due to be freed although the posted count is below the floor. + #[test] + fn kept_backing_freed_at_idle() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(!l.free_due(5 * s, s)); + l.reinit(); + l.posted = 264; + assert!(!l.reclaim_due(1024, 5 * s, s)); + assert!(!l.free_due(s - 1, s)); + assert!(l.free_due(s, s)); + // After the re-initialization and the free, nothing more is due. + l.reinit(); + l.unback_to(l.keep_chunks()); + assert!(!l.free_due(5 * s, s)); + // A heap backed for what it posts plus the reserve is not freed. + l.posted = 1000; + l.unback_to(Ledger::chunks_for(1000)); + assert!(!l.free_due(5 * s, s)); + } + /// A heavy scene, a reclaim that frees the backing, refills during the next heavy scene, /// an idle gap with a small job in it, and the next render must reclaim again. #[test] @@ -622,6 +722,21 @@ mod tests { assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); } + /// A client-boundary re-initialization with 23 chunks backed and no render after it: the + /// kept backing is due to be freed after the long idle, and once freed nothing more is due. + #[test] + fn pending_reinit_frees_kept_backing() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + assert!(!l.pending_free_due(s - 1, s)); + assert!(l.pending_free_due(s, s)); + l.unback_to(l.keep_chunks()); + assert!(!l.pending_free_due(100 * s, s)); + } + #[test] fn grants_only_backed_blocks() { let mut l = Ledger::new(true, 64, 4095); diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 5c3c2ac7452d32..0fa0eaf1b9b436 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -3,10 +3,11 @@ """Run the G15 PB growth answer and its on-demand backing, actual bodies, against a model. The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), -answer_request(), map_pb() and refill_backing(), Pools::soft_max(), map_heap() and -back_chunk(), PbChunk, chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from -vm.rs VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() and -pb_blocks() from params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and +answer_request(), map_pb(), refill_backing(), note_idle(), reclaim_check() and free_backing(), +Pools::soft_max(), map_heap(), free_chunks() and back_chunk(), PbChunk, +chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from vm.rs +VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks(), +pb_blocks(), pb_reclaim(), pb_idle_ns() and pb_reclaim_blocks() from params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and is_cached_noncoherent() from pgtable.rs; and the parameters' defaults from asahi_core.rs. It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() and the DEVCTL @@ -41,6 +42,20 @@ chunk freed and backed again (an idle reclaim) is unmapped and mapped again at publication; - the one-VM rule: a catch-up with a flight of another VM is refused (EPROTO) before anything is mapped; +- the idle reclaim, actual reclaim_check(), free_backing() and note_idle() over a clock: a heavy + client that exits with nothing rendered after it is re-initialized after 50 ms with the backing + kept, and the kept backing is freed after the long idle, its posted count kept until the InitBM; + at reclaim level 1 the backing stays; a recovery's pending re-initialization frees nothing; +- nothing grown since the initialization (an initial posting above half the maximum, or a + reclaim floor below it): no re-initialization; +- the actual reclaim keeps the posted count until the next render's InitBM (a model of the + publication: map_pb() first, then for a render prefix with a re-initialization pending the + ledger reset of Pools::init()): the same client rendering again after 60 ms re-posts the + initial blocks and grows again from the kept backing with nothing allocated; after 1.1 s it + grows again over freshly backed chunks, each mapped before it is posted; a compute-only + submission between the reclaim and the InitBM maps, backs and posts nothing; +- a reclaim whose walk of the VMs fails counts nothing freed; the VM's next publication drops + its stale aliases and releases the pages; - the ledger reset of Pools::init() re-posts from the backing without allocating; - the PB leaf decodes as GPU-only, not cached-noncoherent: dropping an alias is a plain unmap; - Ledger::new() is Ledger::new() as of dba2a110de00 (the carried upstream constructor), capped at @@ -107,7 +122,8 @@ # The two statements of Pools::init() that reset the ledger. LEDGER_RESET = ['self.ledger.reinit();', - 'self.exhausted_logged = false;'] + 'self.exhausted_logged = false;', + 'self.reclaim_pending = false;'] MUTATIONS = { # Answer until the queue is empty, including requests queued by the replies' waits. @@ -167,6 +183,47 @@ 'if self.recovery.reinit && self.engine.flights.is_empty() {'), # Keep a VM's alias of a chunk that was freed and backed again. 'map-without-identity': (VM, ' if core::ptr::eq(&*have.gem, &*gem) {', ' if true {'), + # Stop the reclaim check on a reclaim's pending re-initialization (the kept backing stays). + 'pending-reinit-stops-free': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + # Free the kept backing at reclaim level 1. + 'pending-free-at-level-1': (POOLS, 'if level >= 2 && ledger.pending_free_due(idle_ns, min_idle) {', + 'if ledger.pending_free_due(idle_ns, min_idle) {'), + # Do not record that the pending re-initialization is a reclaim's. + 'reclaim-pending-not-set': (POOLS, ' self.pools.reclaim_pending = true;\n', ''), + # Free for any pending re-initialization, also a recovery's. + 'free-on-recovery-reinit': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + # Re-initialize at an idle with nothing grown (posted above the floor only). + 'reclaim-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns', + 'self.posted > floor && idle_ns >= min_idle_ns'), + # Re-initialize at a client boundary with nothing grown (initial above half the maximum). + 'drain-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS', + 'self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS'), + # Reset the posted count at the idle reclaim instead of at the next render's InitBM. + 'reinit-at-idle-reclaim': (POOLS, ' if level >= 2 && idle {\n freed = self.free_backing();\n', + ' if level >= 2 && idle {\n self.pools.ledger.reinit();\n freed = self.free_backing();\n'), + # Reset the posted count at any reclaim instead of at the next render's InitBM. + 'reinit-at-any-reclaim': (POOLS, ' self.pools.reclaim_pending = true;\n', + ' self.pools.reclaim_pending = true;\n self.pools.ledger.reinit();\n'), + # Count the chunks freed although the VMs could not be walked. + 'freed-without-walk': (POOLS, ' );\n 0\n }\n };\n', + ' );\n dropped\n }\n };\n'), # Catch up without the one-VM check. 'no-activation-guard': (POOLS, '.filter(|f| !core::ptr::eq(&*f.job.vm, vm))', '.filter(|_| false)'), # Back the whole heap when the GPU starts. @@ -202,6 +259,15 @@ 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), 'map-without-identity': ('freed_chunks_are_mapped_again', 'alias of another object'), + 'pending-reinit-stops-free': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), + 'pending-free-at-level-1': ('client_boundary_reinit_at_level_1_keeps_the_backing', 'freed at level 1'), + 'reclaim-pending-not-set': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), + 'free-on-recovery-reinit': ('recovery_reinit_frees_nothing', 'freed for a recovery'), + 'reclaim-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (64 blocks, floor 32)'), + 'drain-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (3000 blocks'), + 'reinit-at-idle-reclaim': ('same_client_render_after_1_1s', 'posted count reset before the InitBM (1500)'), + 'reinit-at-any-reclaim': ('same_client_render_after_60ms', 'posted count reset before the InitBM'), + 'freed-without-walk': ('failed_vm_walk_counts_nothing_freed', 'freed counted with the VM walk failed'), 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), @@ -339,6 +405,14 @@ def consts(text, names): pub static LOG: RefCell> = RefCell::new(Vec::new()); pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); pub static PARAM_BLOCKS: Cell = Cell::new(PB_BLOCKS_DEFAULT); + /// The reclaim parameters (g15_pb_reclaim, g15_pb_idle_ms, g15_pb_reclaim_blocks). + pub static PARAM_RECLAIM: Cell = Cell::new(PB_RECLAIM_DEFAULT); + pub static PARAM_IDLE_MS: Cell = Cell::new(PB_IDLE_MS_DEFAULT); + pub static PARAM_RECLAIM_BLOCKS: Cell = Cell::new(PB_RECLAIM_BLOCKS_DEFAULT); + /// The monotonic clock (engine::now_ns()). + pub static NOW: Cell = Cell::new(1_000_000_000); + /// The VM registry walk (vms.open()) fails this many times. + pub static FAIL_OPEN: Cell = Cell::new(0); /// Failures to inject: the next this many allocations or alias maps fail. pub static FAIL_ALLOC: Cell = Cell::new(0); pub static FAIL_MAP: Cell = Cell::new(0); @@ -394,6 +468,15 @@ def consts(text, names): } } pub static g15_pb_blocks: Blocks = Blocks; + pub struct Cellp(pub &'static std::thread::LocalKey>); + impl Cellp { + pub fn value(&self) -> &'static u32 { + Box::leak(Box::new(self.0.with(|p| p.get()))) + } + } + pub static g15_pb_reclaim: Cellp = Cellp(&crate::PARAM_RECLAIM); + pub static g15_pb_idle_ms: Cellp = Cellp(&crate::PARAM_IDLE_MS); + pub static g15_pb_reclaim_blocks: Cellp = Cellp(&crate::PARAM_RECLAIM_BLOCKS); } pub mod driver { @@ -518,9 +601,31 @@ def consts(text, names): } pub struct Engine { pub flights: Vec, + /// Jobs accepted and not fully published. + pub pending: Vec, + pub frozen: bool, } pub struct Recovery { pub reinit: bool, + pub armed: Option<(usize, u64)>, + pub failed: Option, + } + impl Recovery { + pub fn failed(&self) -> Option { + self.failed + } + } + /// vm.rs's Registry: the VMs whose client handle may still be open. + pub struct Registry { + pub vms: Vec>, + } + impl Registry { + pub fn open(&self) -> Result>> { + if crate::fail(&crate::FAIL_OPEN) { + return Err(ENOMEM); + } + Ok(KVec(self.vms.clone())) + } } pub struct Receipts { pub renders: u32, @@ -560,6 +665,8 @@ def consts(text, names): pub bad: u64, pub waits: u64, pub refills: u64, + pub reclaims: u64, + pub freed: u64, } pub struct Live { pub block_ctl: FwObject, @@ -572,6 +679,8 @@ def consts(text, names): pub live: Live, pub drm: ARef, pub chunks: KVec, + pub idle: growth::IdleClock, + pub reclaim_pending: bool, } impl Pools { pub fn chunk_count(&self) -> usize { @@ -580,12 +689,6 @@ def consts(text, names): pub fn chunk_capacity(&self) -> usize { self.chunks.capacity() } - /// What a reclaim does to the backing (the PB re-initialization included). - pub fn drop_chunks(&mut self, keep: usize) { - self.chunks.truncate(keep); - self.ledger.reinit(); - self.ledger.unback_to(keep as u32); - } /// The backing objects' ids, in heap order. pub fn chunk_ids(&self) -> Vec { self.chunks.iter().map(|c| c.backing.gem.id).collect() @@ -641,6 +744,7 @@ def consts(text, names): pub init: Init, pub engine: Engine, pub recovery: Recovery, + pub vms: Registry, pub renders: Vec, pub fw: FwModel, } @@ -652,6 +756,7 @@ def consts(text, names): let vm = VmState::new(5); let mut f = Self::without_flights(); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + f.vms.vms.push(vm.clone()); f.map_pb(&vm).unwrap(); crate::take(); f @@ -665,6 +770,8 @@ def consts(text, names): pools: Pools { ledger, exhausted_logged: false, + idle: growth::IdleClock::default(), + reclaim_pending: false, stats: PbStats::default(), drm, chunks, @@ -678,8 +785,9 @@ def consts(text, names): }, }, init: Init { control: Some(std::sync::Arc::new(Ctl { epoch: 0x1_0000_0003 })) }, - engine: Engine { flights: Vec::new() }, - recovery: Recovery { reinit: false }, + engine: Engine { flights: Vec::new(), pending: Vec::new(), frozen: false }, + recovery: Recovery { reinit: false, armed: None, failed: None }, + vms: Registry { vms: Vec::new() }, renders: vec![1], fw: FwModel { ctx: 5, @@ -830,6 +938,7 @@ def consts(text, names): /// Growth to the maximum with the firmware asking after every reply, one pass at a time. fn run_to_exhaustion(param: u32) { + defaults(); PARAM.with(|p| p.set(param)); let mut f = Firmware::new(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); @@ -893,6 +1002,7 @@ def consts(text, names): #[test] fn driver_values() { + defaults(); let l = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); assert!(tuning::PB_GROW); assert_eq!(crate::PB_BLOCKS_DEFAULT, 64); @@ -905,6 +1015,7 @@ def consts(text, names): #[test] fn soft_max_bounds() { + defaults(); let f = Firmware::new(); for param in [0, 1, 63, 64, 65, 1023, 1024, 1025, 3071, 3072, 3073, 4094, 4095, 4096, 8192, u32::MAX] { PARAM.with(|p| p.set(param)); @@ -1142,7 +1253,7 @@ def consts(text, names): #[test] fn growth_backs_before_posting() { - PARAM.with(|p| p.set(PARAM_DEFAULT)); + defaults(); let mut f = Firmware::new(); let start = f.pools.ledger.backed as u64; let first_chunks = f.pools.chunk_count(); @@ -1408,42 +1519,321 @@ def consts(text, names): assert_eq!(f.pools.stats.bad, 1); } - /// What a reclaim leaves behind in a VM that was not walked (a closed VM not pruned yet, or - /// one created meanwhile): its aliases of freed chunks go, and chunks backed again at the - /// same GVA are mapped from their new objects at its next publication. + /// Every PB parameter at its default. libtest gives each test its own thread, but a test + /// that sets one starts from the defaults all the same. + fn defaults() { + PARAM.with(|p| p.set(PARAM_DEFAULT)); + crate::PARAM_BLOCKS.with(|b| b.set(crate::PB_BLOCKS_DEFAULT)); + crate::PARAM_RECLAIM.with(|r| r.set(crate::PB_RECLAIM_DEFAULT)); + crate::PARAM_IDLE_MS.with(|r| r.set(crate::PB_IDLE_MS_DEFAULT)); + crate::PARAM_RECLAIM_BLOCKS.with(|r| r.set(crate::PB_RECLAIM_BLOCKS_DEFAULT)); + } + + /// A prefix of `vm` published as engine.rs does it: publish_prefix_into() first catches the + /// VM up (map_pb(), before the flight is added), and a render prefix with a + /// re-initialization pending does the InitBM (Pools::init()'s ledger reset) and clears + /// `reinit` at its publication. `render`: the prefix has renders. + fn publish(f: &mut Firmware, vm: &Arc, render: bool) { + f.map_pb(vm).unwrap(); + if render && f.recovery.reinit { + f.reset_ledger(); + f.recovery.reinit = false; + } + f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + let i = f.engine.flights.len() - 1; + if f.renders.len() <= i { + f.renders.resize(i + 1, 0); + } + f.renders[i] = render as u32; + f.fw.ctx = vm.slot; + f.note_idle(); + } + + /// What a reclaim leaves behind in a VM that was not walked (a closed VM no longer in the + /// registry, whose accepted jobs may still run): its aliases of freed chunks go, and chunks + /// backed again at the same GVA are mapped from their new objects at its next publication. + /// The reclaims are the actual idle reclaim; the posted count stays until the InitBM. #[test] fn freed_chunks_are_mapped_again() { + defaults(); let mut f = Firmware::new(); let a = f.engine.flights[0].job.vm.clone(); - grow_to(&mut f, 300); - let chunks = f.pools.chunk_count(); + grow_to(&mut f, 1100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); assert!(chunks > 2); - // The reclaim frees the chunks above the kept two; A is not walked. + // A's work retires and A leaves the registry; the idle reclaim frees the chunks above + // the kept two without walking A. f.engine.flights.clear(); - f.pools.drop_chunks(2); + f.vms.vms.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); take(); - // B runs and grows the backing again: new objects at the old GVAs. + // B runs: its first render re-initializes; it grows the backing again, new objects at + // the old GVAs. let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.fw.ctx = 6; - f.map_pb(&b).unwrap(); - grow_to(&mut f, 300); + f.vms.vms.push(b.clone()); + publish(&mut f, &b, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 1100); assert_eq!(f.pools.chunk_count(), chunks); // A runs again: its stale aliases are replaced by the new objects'. f.engine.flights.clear(); - f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); take(); - f.map_pb(&a).unwrap(); + publish(&mut f, &a, true); let log = take(); assert_cursor(&f, &a, chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), chunks - 2); - // With fewer chunks than A maps, the ones past the backing go too. - f.pools.drop_chunks(2); + // Another idle reclaim walks B, not A: with fewer chunks than A maps, A's aliases past + // the backing go at its next catch-up. + f.engine.flights.clear(); + let t1 = now(); + check_at(&mut f, t1); + check_at(&mut f, t1 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + assert_cursor(&f, &b, 2); f.map_pb(&a).unwrap(); assert_cursor(&f, &a, 2); } + /// A reclaim whose walk of the VM registry fails: the chunks leave the backing, but a VM's + /// aliases still hold their pages, so nothing is counted freed; the VM drops its stale + /// aliases at its next publication, and only then are the pages released. + #[test] + fn failed_vm_walk_counts_nothing_freed() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1500); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + take(); + crate::FAIL_OPEN.with(|c| c.set(1)); + check_at(&mut f, t0 + 1_000_000_000); + let log = take(); + assert_eq!(f.pools.chunk_count(), 2); + assert_eq!(f.pools.stats.freed, 0, "freed counted with the VM walk failed"); + assert!(log.iter().any(|e| matches!(e, Ev::Log("warn", m) if m.contains("could not walk the VMs")))); + assert!(log.iter().any(|e| matches!(e, Ev::Log("info", m) if m.contains(" 0 chunks freed")))); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), 0, "pages released while aliases hold them"); + assert_eq!(a.heap.list.lock().len(), chunks); + // A's next publication drops the stale aliases: the pages go. + publish(&mut f, &a, true); + let log = take(); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + assert_cursor(&f, &a, 2); + } + + /// One client idles 60 ms after a heavy scene (more than half the maximum posted) and + /// renders again: the client-boundary re-initialization was set at 50 ms with the backing + /// kept, the posted count stays until the render's InitBM, which posts the initial blocks + /// again, and the scene grows again from the kept backing with nothing allocated or mapped. + #[test] + fn same_client_render_after_60ms() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 60_000_000); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!( + (f.pools.ledger.posted, f.pools.chunk_count()), + (posted, chunks), + "posted count reset before the InitBM" + ); + take(); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + assert!(!f.recovery.reinit && !f.pools.reclaim_pending); + grow_to(&mut f, 2100); + let log = take(); + assert_eq!(count(&log, is_backing), 0, "backing changed: {:?}", log.iter().filter(|e| is_backing(e)).collect::>()); + assert_posted_backed(&log, 5, f.pools.ledger.backed as u64); + } + + /// One client idles 1.1 s and renders again, after a scene above the reclaim floor (the + /// idle reclaim at 1 s) or above half the maximum (the client boundary at 50 ms, then the + /// free at 1 s): the backing above the kept chunks was freed and unmapped from the client's + /// VM, the posted count stays until the render's InitBM, and the scene grows again with + /// every chunk backed and mapped before a record or producer names it. + #[test] + fn same_client_render_after_1_1s() { + for target in [1500, 2100] { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, target); + let posted = f.pools.ledger.posted; + f.engine.flights.clear(); + let t0 = now(); + for dt in [0, 50_000_000, 1_000_000_000, 1_100_000_000] { + check_at(&mut f, t0 + dt); + } + assert_eq!(f.pools.chunk_count(), 2, "not freed ({target})"); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM ({target})"); + assert_cursor(&f, &a, 2); + take(); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, target); + let log = take(); + assert_posted_backed(&log, 5, 2 * CHUNK_BLOCKS as u64); + assert_cursor(&f, &a, f.pools.chunk_count()); + } + } + + /// A compute-only submission between a level-2 reclaim and the next render's InitBM: its + /// publication maps nothing, its passes back and post nothing (the posted count is still + /// the old one, above the kept backing), and a growth request naming its context is left + /// unanswered; once it retires, the render after it re-initializes as usual. + #[test] + fn compute_between_reclaim_and_initbm() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1500); + let posted = f.pools.ledger.posted; + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + let bad = f.pools.stats.bad; + take(); + publish(&mut f, &a, false); + f.queue(); + for _ in 0..4 { + f.pass(); + } + let log = take(); + let touched: Vec<&Ev> = log + .iter() + .filter(|e| is_backing(e) || matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_))) + .collect(); + assert!(touched.is_empty(), "compute-only flight backed or posted: {touched:?}"); + assert!(f.recovery.reinit && f.pools.ledger.posted == posted); + assert_eq!(f.pools.stats.bad, bad + 1); + // The compute retires; the check finds nothing more to do; the next render re-inits. + f.engine.flights.clear(); + check_at(&mut f, t0 + 2_000_000_000); + assert_eq!(count(&take(), is_backing), 0); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 300); + assert_posted_backed(&take(), 5, 2 * CHUNK_BLOCKS as u64); + } + + fn now() -> u64 { + crate::NOW.with(|n| n.get()) + } + + /// The idle clock at `t` and the reclaim check at that time, as a job arrival or an idle + /// poll pass runs it. + fn check_at(f: &mut Firmware, t: u64) { + crate::NOW.with(|n| n.set(t)); + f.note_idle(); + f.reclaim_check(); + } + + /// A heavy client (more than half the maximum posted) exits and nothing renders after it: + /// the 50 ms client-boundary re-initialization keeps the backing, and a check after the + /// long idle frees the backing above the kept chunks. The posted count stays until the next + /// render's InitBM. + #[test] + fn client_boundary_reinit_frees_without_a_render() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(posted > 2047 && chunks > 2); + // The pass that retires the last flight notes the idle. + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + take(); + check_at(&mut f, t0 + 50_000_000); + assert!(f.recovery.reinit && f.pools.chunk_count() == chunks, "client-boundary re-initialization"); + check_at(&mut f, t0 + 999_999_999); + assert_eq!(f.pools.chunk_count(), chunks, "freed before the long idle"); + check_at(&mut f, t0 + 1_000_000_000); + let log = take(); + assert_eq!(f.pools.chunk_count(), 2, "kept backing not freed: {} chunks", f.pools.chunk_count()); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!(f.pools.stats.reclaims, 1); + assert_eq!(f.pools.ledger.backed, 2 * CHUNK_BLOCKS); + assert_eq!(f.pools.stats.freed, (chunks - 2) as u64); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + // Later checks find nothing more to do. + check_at(&mut f, t0 + 100_000_000_000); + assert!(take().is_empty()); + } + + /// With g15_pb_reclaim 1 the client-boundary re-initialization keeps the backing for good. + #[test] + fn client_boundary_reinit_at_level_1_keeps_the_backing() { + defaults(); + crate::PARAM_RECLAIM.with(|r| r.set(1)); + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 50_000_000); + assert!(f.recovery.reinit); + check_at(&mut f, t0 + 5_000_000_000); + assert_eq!(f.pools.chunk_count(), chunks, "freed at level 1"); + assert_eq!(f.pools.stats.freed, 0); + } + + /// Nothing grown since the initialization: an initial posting above half the maximum + /// (g15_pb_blocks 3000), or a floor below it (g15_pb_reclaim_blocks 32): no + /// re-initialization at any idle. + #[test] + fn nothing_grown_reinitializes_nothing() { + for (blocks, floor) in [(64, 32), (3000, 1024)] { + defaults(); + crate::PARAM_BLOCKS.with(|b| b.set(blocks)); + crate::PARAM_RECLAIM_BLOCKS.with(|b| b.set(floor)); + let mut f = Firmware::new(); + assert_eq!(f.pools.ledger.posted, blocks); + f.engine.flights.clear(); + let t0 = now(); + for dt in [0, 50_000_000, 1_000_000_000, 5_000_000_000] { + check_at(&mut f, t0 + dt); + } + assert!( + !f.recovery.reinit && f.pools.stats.reclaims == 0, + "re-initialization with nothing grown ({blocks} blocks, floor {floor})" + ); + } + } + + /// A recovery's re-initialization (not a reclaim's) is left alone: nothing freed. + #[test] + fn recovery_reinit_frees_nothing() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + f.recovery.reinit = true; + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 5_000_000_000); + assert_eq!(f.pools.chunk_count(), chunks, "freed for a recovery's re-initialization"); + assert_eq!(f.pools.stats.reclaims, 0); + } + #[test] fn catch_up_refuses_flights_of_another_vm() { let mut f = Firmware::new(); @@ -1533,9 +1923,13 @@ def run(mutation): bodies = [item(pools, s) for s in (' pub(crate) fn answer_growth(&mut self) -> Result {', ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', ' pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result {', - ' pub(crate) fn refill_backing(&mut self) {')] + ' pub(crate) fn refill_backing(&mut self) {', + ' pub(crate) fn note_idle(&mut self) {', + ' pub(crate) fn reclaim_check(&mut self) {', + ' fn free_backing(&mut self) -> usize {')] soft_max = item(pools, ' pub(crate) fn soft_max(&self) -> u32 {') pools_methods = (item(pools, ' fn map_heap(&self, vm: &VmState) -> Result {') + '\n' + + item(pools, ' fn free_chunks(&mut self, keep: usize) -> usize {') + '\n' + item(pools, ' fn back_chunk(&mut self) -> Result> {')) chunk = item(pools, 'struct PbChunk {') + '\n' + item(pools, 'impl PbChunk {') chunk_geometry = (re.search(r'^pub\(crate\) const CHUNK_BYTES: usize = [^;]+;', pools, re.M).group(0) + '\n' + @@ -1569,8 +1963,16 @@ def run(mutation): if init.count(line) != 1: raise SystemExit('Pools::init() does not hold: ' + line) reset = '\n'.join('p.' + line[len('self.'):] for line in LEDGER_RESET) - pb_max_blocks = (item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> u32 {') + '\n' + - item(src[PARAMS], 'pub(crate) fn pb_blocks() -> u32 {')) + pb_max_blocks = '\n'.join(item(src[PARAMS], f'pub(crate) fn {name}() -> ') + for name in ('pb_max_blocks', 'pb_blocks', 'pb_reclaim', 'pb_idle_ns', + 'pb_reclaim_blocks')) + reclaim_defaults = {} + for param, const in (('g15_pb_reclaim', 'PB_RECLAIM_DEFAULT'), ('g15_pb_idle_ms', 'PB_IDLE_MS_DEFAULT'), + ('g15_pb_reclaim_blocks', 'PB_RECLAIM_BLOCKS_DEFAULT')): + m = re.search(rf'{param}: u32 \{{\s*default: (\d+),', src[CORE]) + if not m: + raise SystemExit(f'{param} default not found') + reclaim_defaults[const] = int(m.group(1)) m = re.search(r'g15_pb_max_blocks: u32 \{\s*default: (\d+),', src[CORE]) if not m: raise SystemExit('g15_pb_max_blocks default not found') @@ -1595,6 +1997,7 @@ def run(mutation): (dir / Path(path).name).write_text(src[path]) code = (PREFIX + f'pub const PARAM_DEFAULT: u32 = {param_default};\n' + f'pub const PB_BLOCKS_DEFAULT: u32 = {blocks_default};\n' + + ''.join(f'pub const {k}: u32 = {v};\n' for k, v in reclaim_defaults.items()) + 'pub mod g15 {\n' ' pub mod fw {\n' f' #[path = "{dir / "channels.rs"}"] pub mod channels;\n' @@ -1611,6 +2014,7 @@ def run(mutation): ' }\n' ' pub mod pgtable {\n' + pg_consts + '\n' + prot + '\n }\n' ' pub mod boot {\n pub fn eproto() -> crate::Error { crate::EPROTO }\n }\n' + ' pub mod engine {\n pub fn now_ns() -> u64 { crate::NOW.with(|n| n.get()) }\n }\n' ' /// The checked backing allocation, failing where a test injects it.\n' ' pub mod object {\n' ' use crate::prelude::*;\n' From ae93621adf50dfc39c549b25642e22dfc95cf7ac Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:57 +0530 Subject: [PATCH 08/11] drm/asahi: g15: m3: bound the CDM stream walk by bytes, not by 8192 records The CDM gate walks a compute stream from its base through its links and refuses it after 8192 records. A legitimate stream can be longer. On the J516C, the three dEQP-VK multi_draw.mosaic cases with max_draws and tessellation (1024 tessellated draws, several launches and barriers each) were refused: "rejecting CDM record budget record", then EINVAL from the submit. The gate is shared by the T6030 and the T6031. It already admits at most 64 links, and Mesa's encoder links to a new 64 KiB block when one fills, so a stream that fits the link budget is at most 65 blocks. Bound the walk by the bytes of its records instead: at most (64 + 1) * 64 KiB (4160 KiB) of stream, whatever the mix of 4-byte barriers and 24- to 40-byte launches. A stream without TERMINATE still ends at that bound or at the link budget, so the walk stays bounded: at most 1064960 record reads of 4 bytes. A walk does not check again the last page it found good, for a program or for an indirect launch's 12 argument bytes, and it makes at most 16384 page checks; one more is refused as "CDM page check budget". The walk holds no device lock: the queue's mutex covers the SUBMIT, and the VM's lock is taken per check (about 4 us on the J516C) and per 256-byte read (about 5 us). Without the cache and the bound, a stream of 24-byte indirect launches with distinct argument pages made one check per launch: about 0.8 s per compute command and about 51 s per 64-command SUBMIT. Now it is at most about 149 ms per command and about 9.5 s per SUBMIT. Mesa's indirect arguments move to a new page every few dispatches: 40000 dispatches made 41410 checks in 224 ms before and 6848 in 94 ms after, and the largest stream the link budget admits (47000 dispatches) makes 8047. The 608 dEQP-VK multi_draw.mosaic max_draws with_tess cases pass. Host tests (asahi-g15 selftest): 10240 barriers and a terminator are admitted; a terminator in the last word of the budget is admitted and one past it refused; a stream of the budget without a terminator is refused; 64 links are admitted and 65 refused; 1000 launches in one argument page make 2 page checks; a program and 16383 argument pages are admitted and one more page is refused without its check; 170000 launches over 16 argument pages are admitted with 17 checks. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/asahi/g15/gate.rs | 92 +++++++++++++++---- .../selftests/drivers/gpu/asahi-g15/run.py | 92 ++++++++++++++++++- 2 files changed, 164 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/gate.rs b/drivers/gpu/drm/asahi/g15/gate.rs index f41b2e3f75f57c..a6fb20bf156b5c 100644 --- a/drivers/gpu/drm/asahi/g15/gate.rs +++ b/drivers/gpu/drm/asahi/g15/gate.rs @@ -3,11 +3,13 @@ //! Stream admission gates (safety nets for userspace encoders). //! //! VDM: the root stream's first word is 0x21000960. CDM walk from `cdm_ctrl_stream_base` through -//! links until TERMINATE (0x40000000), at most 8192 records and 64 links (a longer or cyclic -//! stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; +//! links until TERMINATE (0x40000000), at most 64 links and [`CDM_MAX_BYTES`] of records (a +//! longer or cyclic stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; //! link 0x2000XXXX; direct launch (40 B, the G15 layout), indirect launches 36 B //! (`w0 & 0xff7fffff == 0x08080000`) and 24 B (0x10080000); every program VA inside the queue's -//! 4 GiB USC window, 64-aligned, first 64 bytes readable. Client memory is read through the VM's +//! 4 GiB USC window, 64-aligned, first 64 bytes readable; an indirect launch's 12 argument bytes +//! readable. A program or argument page found good is not checked again in the same walk, and a +//! walk checks at most [`CDM_MAX_PAGE_CHECKS`] pages. Client memory is read through the VM's //! page tables with a point-of-coherency clean+invalidate of the lines ([`mmu::Vm::read_user`]: //! the GPU may have written it). `g15_stream_gate=0` disables both gates (encoder bring-up only). @@ -24,8 +26,19 @@ const CDM_LINK: u32 = 0x20000000; const CDM_DIRECT: u32 = 0x00080000; const CDM_INDIRECT_SIZES: u32 = 0x08080000; const CDM_INDIRECT: u32 = 0x10080000; -const CDM_MAX_RECORDS: u32 = 8192; const CDM_MAX_LINKS: u32 = 64; +/// The stream block the encoder links to when one fills (Mesa's 64 KiB control-stream blocks). +const CDM_BLOCK_BYTES: u64 = 0x10000; +/// Stream bytes walked at most: the root block and one block per admitted link. A stream of +/// that size holds every record the encoder can write before the link budget (a record count +/// does not: the 1024 tessellated draws of one multi-draw write more than 8192). +const CDM_MAX_BYTES: u64 = (CDM_MAX_LINKS as u64 + 1) * CDM_BLOCK_BYTES; +/// Page checks (a program's or an indirect launch's arguments, each under the VM lock) a walk +/// makes at most. The last page found good is not checked again, but the encoder's indirect +/// arguments move to a new page every few launches: a stream at the link budget makes about +/// 8100. Twice that leaves room; without a bound, a stream of distinct pages would make one per +/// 24-byte launch, about 177000 in the byte budget. +const CDM_MAX_PAGE_CHECKS: u32 = 16384; /// Rejections logged per boot. const MAX_REPORTS: u32 = 32; @@ -86,19 +99,52 @@ impl Window { } } +/// The pages a walk has found good and the page checks it has made. +struct Checked { + program: Option, + args: Option, + count: u32, +} + +/// One page check of `[start, end)` under the walk's page budget: ENOSPC once the budget is +/// spent. +fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result { + if *count >= CDM_MAX_PAGE_CHECKS { + return Err(ENOSPC); + } + *count += 1; + cover(vm, start, end, true, false) +} + /// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable. -/// `checked` caches the last proven program page. -fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Option) -> Result { +/// The last proven program page is not checked again. +fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Result { if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 { return Err(EINVAL); } // 64-aligned: the block never crosses its page. let page = address & !(mmu::UAT_PGMSK as u64); - if *checked == Some(page) { + if checked.program == Some(page) { return Ok(()); } - cover(vm, address, address + 64, true, false)?; - *checked = Some(page); + check(vm, address, address + 64, &mut checked.count)?; + checked.program = Some(page); + Ok(()) +} + +/// An indirect launch's 12 argument bytes: readable. The last proven argument page is not +/// checked again; arguments that cross a page are checked each time. +fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result { + let end = address.checked_add(12).ok_or(EINVAL)?; + let page = address & !(mmu::UAT_PGMSK as u64); + let within = (end - 1) & !(mmu::UAT_PGMSK as u64) == page; + if within && checked.args == Some(page) { + return Ok(()); + } + check(vm, address, end, &mut checked.count)?; + if within { + checked.args = Some(page); + } Ok(()) } @@ -128,15 +174,19 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { len: 0, bytes: [0; 256], }; - let mut checked = None; + let mut checked = Checked { + program: None, + args: None, + count: 0, + }; let mut va = base; let mut links = 0; - let mut records = 0; - while records < CDM_MAX_RECORDS { - records += 1; + let mut walked = 0; + while walked < CDM_MAX_BYTES { let w = window.words(vm, va, 1)?; if CDM_BARRIERS.contains(&w[0]) { va += 4; + walked += 4; continue; } if w[0] == CDM_TERMINATE { @@ -152,6 +202,7 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { if va & 3 != 0 { return Err(reject("CDM link", va, &w)); } + walked += 8; continue; } if w[0] == CDM_DIRECT { @@ -162,9 +213,13 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { } match program(vm, usc, address, &mut checked) { Err(error) if error == EINVAL => return Err(reject("CDM program", va, &w)), + Err(error) if error == ENOSPC => { + return Err(reject("CDM page check budget", va, &w)) + } result => result?, } va += 40; + walked += 40; continue; } let kind = w[0] & 0xff7fffff; @@ -182,15 +237,20 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { return Err(reject("CDM indirect launch", va, &w)); } let result = program(vm, usc, address, &mut checked) - .and_then(|()| cover(vm, indirect, indirect + 12, true, false)); + .and_then(|()| arguments(vm, indirect, &mut checked)); match result { Err(error) if error == EINVAL => return Err(reject("CDM indirect", va, &w)), + Err(error) if error == ENOSPC => { + return Err(reject("CDM page check budget", va, &w)) + } result => result?, } - va += if sizes { 36 } else { 24 }; + let size = if sizes { 36 } else { 24 }; + va += size; + walked += size; continue; } return Err(reject("CDM", va, &w)); } - Err(reject("CDM record budget", va, &[])) + Err(reject("CDM byte budget", va, &[])) } diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 35c96a7f36176c..861290ef240986 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -102,6 +102,7 @@ pub type Result = std::result::Result; pub const EINVAL: Error = -22; pub const EFAULT: Error = -14; + pub const ENOSPC: Error = -28; pub use crate::{pr_warn, static_assert}; } #[macro_export] @@ -124,8 +125,19 @@ } mod command { use crate::{mmu, prelude::*}; + use std::cell::Cell; pub const USC_WINDOW: u64 = 1 << 32; + /// Addresses from here on are mapped and readable without backing in the mock VM. + pub const SYNTHETIC: u64 = 1 << 40; + thread_local! { + /// cover() calls: the page checks the walk made. + pub static COVERS: Cell = Cell::new(0); + } pub fn cover(vm: &mmu::Vm, start: u64, end: u64, _: bool, _: bool) -> Result { + COVERS.with(|c| c.set(c.get() + 1)); + if start >= SYNTHETIC { + return Ok(()); + } let mut bytes = vec![0; usize::try_from(end - start).map_err(|_| EINVAL)?]; vm.read_user(start, &mut bytes) } @@ -151,16 +163,88 @@ fn cyclic_links_are_rejected() { assert_eq!(gate::cdm(&vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); } + /// Words of 4 bytes that fill the byte budget: the root block and one per link. + const BUDGET_WORDS: usize = 65 * 0x10000 / 4; #[test] - fn record_budget_requires_a_terminator() { - assert_eq!(gate::cdm(&vm(&vec![0x60000160; 8192]), 0, 0), Err(EINVAL)); + fn byte_budget_requires_a_terminator() { + assert_eq!(gate::cdm(&vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); } #[test] fn final_record_may_terminate() { - let mut words = vec![0x60000160; 8192]; - words[8191] = 0x40000000; + let mut words = vec![0x60000160; BUDGET_WORDS + 1]; + words[BUDGET_WORDS - 1] = 0x40000000; + assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + words[BUDGET_WORDS - 1] = 0x60000160; + words[BUDGET_WORDS] = 0x40000000; + assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + } + #[test] + fn more_than_8192_records_are_admitted() { + // 1024 draws of ten records each, as a tessellated multi-draw writes them. + let mut words = vec![0x60000160; 10240]; + words.push(0x40000000); + assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + } + #[test] + fn link_budget_still_bounds_the_walk() { + // 65 links, each to the next: the 65th is refused. + let mut words = Vec::new(); + for i in 0..65u32 { + words.extend([0x20000000, (i + 1) * 8]); + } + words.push(0x40000000); + assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + let mut words = Vec::new(); + for i in 0..64u32 { + words.extend([0x20000000, (i + 1) * 8]); + } + words.push(0x40000000); assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); } + /// The USC window of these cases and its first program, both in the mock's mapped range. + const USC: u64 = crate::command::SYNTHETIC; + /// A 24-byte indirect launch of the window's first program with its arguments at `args`. + fn indirect(args: u64) -> [u32; 6] { + [0x10080000, ((USC >> 38) as u32) << 22, (USC >> 6) as u32, + 0x40000000 | ((USC >> 40) & 0xff) as u32, (args >> 32) as u32 & 0xffff, args as u32] + } + /// `n` indirect launches, launch `i` with its arguments at `args(i)`, then TERMINATE. + fn launches(n: usize, args: impl Fn(usize) -> u64) -> Vec { + let mut words: Vec = (0..n).flat_map(|i| indirect(args(i))).collect(); + words.push(0x40000000); + words + } + fn covers(words: &[u32]) -> (Result, u32) { + crate::command::COVERS.with(|c| c.set(0)); + let r = gate::cdm(&vm(words), USC, 0); + (r, crate::command::COVERS.with(|c| c.get())) + } + const PAGE: u64 = mmu::UAT_PGSZ as u64; + #[test] + fn repeated_argument_pages_are_checked_once() { + // 1000 launches with their arguments in one page: the program page and that page. + let base = 2 * USC; + assert_eq!(covers(&launches(1000, |i| base + (i as u64 % 100) * 12)), (Ok(()), 2)); + // Alternating between two pages: one check per change of page. + assert_eq!(covers(&launches(10, |i| base + (i as u64 % 2) * PAGE)), (Ok(()), 11)); + // Arguments that cross a page are checked each time. + assert_eq!(covers(&launches(3, |_| base + PAGE - 8)), (Ok(()), 4)); + } + #[test] + fn page_check_budget_bounds_the_walk() { + let base = 2 * USC; + // The program page and 16383 argument pages: 16384 checks, admitted. + assert_eq!(covers(&launches(16383, |i| base + i as u64 * PAGE)), (Ok(()), 16384)); + // One more page: refused at the 16385th check, which is not made. + assert_eq!(covers(&launches(16384, |i| base + i as u64 * PAGE)), (Err(EINVAL), 16384)); + } + #[test] + fn a_large_stream_of_reused_pages_is_admitted() { + // 170000 launches (4 MB, inside the byte budget) over 16 argument pages. + let base = 2 * USC; + let words = launches(170000, |i| base + (i as u64 / 10625) * PAGE + (i as u64 % 1000) * 12); + assert_eq!(covers(&words), (Ok(()), 17)); + } #[test] fn malformed_and_unreadable_streams_are_rejected() { assert_eq!(gate::cdm(&vm(&[0xffffffff]), 0, 0), Err(EINVAL)); From c562b82dd6654017e6404d83b00d809ae298cc04 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:14:08 +0530 Subject: [PATCH 09/11] drm/asahi: g15: select the Rust firmware loader abstractions The G15 backend loads the T6031 board images with kernel::firmware (g15/board/blob.rs), which exists only with CONFIG_RUST_FW_LOADER_ABSTRACTIONS. DRM_ASAHI_G15 does not select it: the symbol comes only from APPLE_AOP, APPLE_SEP or nova-core, so a configuration without them fails to build the driver. Select RUST_FW_LOADER_ABSTRACTIONS from DRM_ASAHI_G15. It depends only on RUST, which DRM_ASAHI already requires, and selects FW_LOADER. Built with APPLE_AOP and APPLE_SEP off: the option resolves to y and drivers/gpu/drm/asahi builds. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/asahi/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 958600a91b06bd..9183073cd92236 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -42,6 +42,7 @@ config DRM_ASAHI_G15 depends on DRM_APPLE=y || (DRM_APPLE=m && DRM_ASAHI=m) select APPLE_T6030_GPU_PMP_LINK select CRYPTO_LIB_SHA256 + select RUST_FW_LOADER_ABSTRACTIONS default n help Enable the native G15 backend for explicitly admitted M3 Pro/Max From aa142eed3a7cea0d48f5b98a8f7e1f7518b5aaf1 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:14:52 +0530 Subject: [PATCH 10/11] drm/asahi: g15: m3: run the PB reclaim check after the idle time without new work The reclaim check runs in poll passes and when a job arrives, but an idle engine runs no poll pass: the pass that retires the last flight is not re-armed. On the J516C, a client-boundary re-initialization kept 23 chunks (368 MiB) for a client that then posted 264 blocks. With no new job they were still held 15 s later, and only the next job's arrival freed 21 of them. When a poll pass ends with the engine idle, compute how much longer the engine must stay idle before the reclaim check would act (Ledger::reclaim_delay(): the idle reclaim at g15_pb_idle_ms, the free of kept backing at level 2, or the client-boundary re-initialization). Then queue the poll work for that delay. The pass it runs takes the device lock and runs the idle path: note_idle(), then reclaim_check(). Nothing is armed while busy, while a recovery or its re-initialization is pending, or when nothing can become due. While a reclaim's re-initialization is pending (a client-boundary one keeps the backing), the poll is armed for the rest of g15_pb_idle_ms at level 2 (Ledger::pending_free_delay()), not for the expired 50 ms, so the check frees the kept backing of a heavy client that exits with no render after it. A job's arrival re-arms the poll work for its publication, which replaces the expiry. The delay mirrors the checks, also in requiring growth since the initialization. This applies to the T6030 and the T6031, which share this code; it is untested on the T6030 (g15_pb_reclaim=0 turns the reclaim off, and g15_pb_blocks=1024 restores the old start). Host test (pools/growth.rs): over levels, floors, idle times, posted counts and backing, the delay is the first idle time at which reclaim_due(), free_due() or drain_reinit_due() holds. None of them holds one nanosecond before it, and none ever holds when there is no delay. The J516C case gives g15_pb_idle_ms at level 2 and nothing at levels 0 and 1. The growth harness also runs reclaim_delay_ns() and drives the idle poll work as poll.rs arms it: a heavy client that exits with nothing rendered after it wakes the poll at 54 ms (the re-initialization) and at 1004 ms (the free), and nothing is armed after; an idle reclaim arms nothing once its re-initialization is pending; nothing is armed at level 1, for a recovery's re-initialization, or when nothing grew. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/asahi/g15/poll.rs | 12 +- drivers/gpu/drm/asahi/g15/pools.rs | 31 ++++ drivers/gpu/drm/asahi/g15/pools/growth.rs | 133 ++++++++++++++++- tools/asahi/g15-pb-growth-tests.py | 165 ++++++++++++++++++++-- 4 files changed, 324 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 36f27b3aa58568..751bccd07b119e 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -23,7 +23,7 @@ use kernel::{ hrtimer::{ HrTimerCallback, HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode, }, - Delta, Jiffies, Monotonic, + msecs_to_jiffies, Delta, Jiffies, Monotonic, }, workqueue::RawWorkItem, }; @@ -85,7 +85,15 @@ impl Shared { let evented = self.evented.swap(false, Ordering::AcqRel); match firmware.poll(evented, &mut done) { Ok(true) => self.rearm(firmware.engine.chan_wait), - Ok(false) => {} + // Idle: nothing polls until new work arrives, so a PB reclaim that comes + // due with more idle time gets one more pass then. A job's arrival brings + // the poll work forward (`rearm`), which replaces this expiry. + Ok(false) => { + if let Some(ns) = firmware.reclaim_delay_ns() { + let ms = ns.div_ceil(1_000_000).min(u32::MAX as u64) as u32; + self.mod_poll(msecs_to_jiffies(ms) + 1); + } + } Err(error) => self.fail_stop(error), } } diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index f03b196760b6ac..3364b136b9287f 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -871,6 +871,37 @@ impl Firmware { Ok(()) } + /// How long the idle engine must stay idle before [`Self::reclaim_check`] acts, if it can + /// act without new work ([`growth::Ledger::reclaim_delay`], or with a reclaim's + /// re-initialization pending [`growth::Ledger::pending_free_delay`]); `None` while busy, + /// while a recovery or its re-initialization is pending, or when nothing would become due. + /// An idle engine runs no poll pass by itself, so the poll work is armed for this delay. + pub(crate) fn reclaim_delay_ns(&self) -> Option { + if !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || (self.recovery.reinit && !self.pools.reclaim_pending) + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return None; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + if self.recovery.reinit { + return self.pools.ledger.pending_free_delay( + super::params::pb_reclaim(), + idle_ns, + super::params::pb_idle_ns(), + ); + } + self.pools.ledger.reclaim_delay( + super::params::pb_reclaim(), + super::params::pb_reclaim_blocks(), + idle_ns, + super::params::pb_idle_ns(), + ) + } + /// Processes one growth request: `Ok(false)` when it waits for backing. fn answer_request(&mut self, request: &[u32; 14]) -> Result { let ctl = self.init.control.clone().ok_or(ENODEV)?; diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index c98aef060f94e8..fef40a6d3a73e3 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -179,6 +179,22 @@ impl Ledger { idle_ns >= min_idle_ns && self.backed > self.keep_chunks() * CHUNK_BLOCKS } + /// How much longer the engine must stay idle, from `idle_ns`, before + /// [`Self::pending_free_due`] with `level`; `None` when it cannot become due (level below + /// 2, or nothing above the kept chunks). 0 when it is due now. + pub(crate) const fn pending_free_delay( + &self, + level: u32, + idle_ns: u64, + min_idle_ns: u64, + ) -> Option { + if level >= 2 && self.backed > self.keep_chunks() * CHUNK_BLOCKS { + Some(min_idle_ns.saturating_sub(idle_ns)) + } else { + None + } + } + /// Whether a short idle re-initializes the heap: more than half the maximum posted and /// idle for [`DRAIN_IDLE_NS`]. Back-to-back heavy scenes (a new client every few hundred /// ms) never reach `g15_pb_idle_ms`, and posted blocks never come back while a PB lives, so @@ -189,6 +205,36 @@ impl Ledger { self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS } + /// How much longer the engine must stay idle, from `idle_ns`, before a reclaim with + /// `level` is due ([`Self::reclaim_due`] with `floor` and `min_idle_ns`, [`Self::free_due`] + /// at level 2, or [`Self::drain_reinit_due`]); `None` when none can become due while the + /// ledger stays as it is. 0 when one is due now. + pub(crate) const fn reclaim_delay( + &self, + level: u32, + floor: u32, + idle_ns: u64, + min_idle_ns: u64, + ) -> Option { + if level == 0 { + return None; + } + let grown = self.posted > self.initial; + let mut due = u64::MAX; + if grown && self.posted > self.max / 2 { + due = DRAIN_IDLE_NS; + } + let free = level >= 2 && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS; + if ((grown && self.posted > floor) || free) && min_idle_ns < due { + due = min_idle_ns; + } + if due == u64::MAX { + None + } else { + Some(due.saturating_sub(idle_ns)) + } + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -722,8 +768,84 @@ mod tests { assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); } + /// The delay names exactly the idle time at which the reclaim check first acts: nothing is + /// due one nanosecond before it, something is due at it, and nothing ever becomes due + /// when it is `None`. + #[test] + fn reclaim_delay_matches_the_checks() { + let s = 1_000_000_000; + let due = |l: &Ledger, level: u32, floor: u32, idle: u64, min_idle: u64| { + level != 0 + && (l.reclaim_due(floor, idle, min_idle) + || (level >= 2 && l.free_due(idle, min_idle)) + || l.drain_reinit_due(idle)) + }; + for level in 0..=2 { + for floor in [0, 64, 1024, 4095] { + for min_idle in [DRAIN_IDLE_NS / 2, 100_000_000, s] { + for posted in [64, 200, 1024, 1025, 2047, 2048, 2900, 4095] { + for chunks in [2, 3, 9, 23, MAX_CHUNKS] { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(chunks); + l.posted = posted; + match l.reclaim_delay(level, floor, 0, min_idle) { + None => { + for idle in [0, DRAIN_IDLE_NS, min_idle, 100 * s] { + assert!(!due(&l, level, floor, idle, min_idle)); + } + } + Some(d) => { + assert!(due(&l, level, floor, d, min_idle)); + assert!(d == 0 || !due(&l, level, floor, d - 1, min_idle)); + // Counted from an idle already begun. + assert_eq!( + l.reclaim_delay(level, floor, d / 2, min_idle), + Some(d - d / 2) + ); + assert_eq!( + l.reclaim_delay(level, floor, d + s, min_idle), + Some(0) + ); + } + } + } + } + } + } + } + // The case of the J516C run: a client-boundary re-initialization left 23 chunks for + // 264 posted blocks; the free is due after g15_pb_idle_ms of idle. + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 264; + assert_eq!(l.reclaim_delay(2, 1024, 0, s), Some(s)); + assert_eq!(l.reclaim_delay(1, 1024, 0, s), None); + assert_eq!(l.reclaim_delay(0, 1024, 0, s), None); + // A desktop at the initial posting with the kept chunks: nothing to wait for. + l.unback_to(l.keep_chunks()); + l.posted = 64; + assert_eq!(l.reclaim_delay(2, 1024, 0, s), None); + // Nothing grown, with an initial posting above half the maximum or above the floor: + // nothing armed. + let mut l = Ledger::new(true, 3000, 4095); + l.unback_to(Ledger::chunks_for(3000)); + assert_eq!( + l.reclaim_delay(2, 1024, 0, s), + None, + "armed with nothing grown" + ); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert_eq!( + l.reclaim_delay(2, 32, 0, s), + None, + "armed with nothing grown" + ); + } + /// A client-boundary re-initialization with 23 chunks backed and no render after it: the - /// kept backing is due to be freed after the long idle, and once freed nothing more is due. + /// kept backing is due to be freed after the long idle, measured from the start of the idle, + /// and once freed nothing more is due. #[test] fn pending_reinit_frees_kept_backing() { let s = 1_000_000_000; @@ -731,10 +853,19 @@ mod tests { l.unback_to(23); l.posted = 2800; assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // The poll after the 50 ms re-initialization waits for the rest of the long idle. + assert_eq!( + l.pending_free_delay(2, DRAIN_IDLE_NS, s), + Some(s - DRAIN_IDLE_NS) + ); assert!(!l.pending_free_due(s - 1, s)); assert!(l.pending_free_due(s, s)); + assert_eq!(l.pending_free_delay(2, s + 1, s), Some(0)); + assert_eq!(l.pending_free_delay(1, 0, s), None); + assert_eq!(l.pending_free_delay(0, 0, s), None); l.unback_to(l.keep_chunks()); assert!(!l.pending_free_due(100 * s, s)); + assert_eq!(l.pending_free_delay(2, 0, s), None); } #[test] diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 0fa0eaf1b9b436..e7f350d5c7f877 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -3,17 +3,17 @@ """Run the G15 PB growth answer and its on-demand backing, actual bodies, against a model. The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), -answer_request(), map_pb(), refill_backing(), note_idle(), reclaim_check() and free_backing(), -Pools::soft_max(), map_heap(), free_chunks() and back_chunk(), PbChunk, -chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from vm.rs -VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks(), -pb_blocks(), pb_reclaim(), pb_idle_ns() and pb_reclaim_blocks() from params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and -is_cached_noncoherent() from pgtable.rs; and the parameters' defaults from asahi_core.rs. It -includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model -of the firmware objects, the GEM backing allocation, the shared layer's map_at() and the DEVCTL -wait, which logs every allocation, alias map, unmap, free, block record, barrier, producer -store, reply and log line in order. Allocation and alias map failures can be injected. Source -checks: publish_prefix_into() starts with the PB catch-up, and pools.rs asserts the PB leaf +answer_request(), map_pb(), refill_backing(), note_idle(), reclaim_check(), free_backing() and +reclaim_delay_ns(), Pools::soft_max(), map_heap(), free_chunks() and back_chunk(), PbChunk, +chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from vm.rs VmState::map_one(), +map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks(), pb_blocks(), pb_reclaim(), +pb_idle_ns() and pb_reclaim_blocks() from params.rs; the PB sizes, tuning and PB leaf word from +hw.rs; Prot::from_pte() and is_cached_noncoherent() from pgtable.rs; and the parameters' defaults +from asahi_core.rs. It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies +run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() +and the DEVCTL wait, which logs every allocation, alias map, unmap, free, block record, barrier, +producer store, reply and log line in order. Allocation and alias map failures can be injected. +Source checks: publish_prefix_into() starts with the PB catch-up, and pools.rs asserts the PB leaf GPU-only at compile time. What it shows: @@ -46,8 +46,11 @@ client that exits with nothing rendered after it is re-initialized after 50 ms with the backing kept, and the kept backing is freed after the long idle, its posted count kept until the InitBM; at reclaim level 1 the backing stays; a recovery's pending re-initialization frees nothing; + driven as the idle poll work arms it (reclaim_delay_ns()), the same exit wakes at 50 ms and at + the long idle and arms nothing after; an idle reclaim arms nothing once its re-initialization + is pending; - nothing grown since the initialization (an initial posting above half the maximum, or a - reclaim floor below it): no re-initialization; + reclaim floor below it): no re-initialization and nothing armed; - the actual reclaim keeps the posted count until the next render's InitBM (a model of the publication: map_pb() first, then for a render prefix with a re-initialization pending the ledger reset of Pools::init()): the same client rendering again after 60 ms re-posts the @@ -209,12 +212,49 @@ ' || self.engine.frozen\n' ' || self.recovery.failed().is_some()\n' ' {\n return;\n'), + # Both checks stop on any pending re-initialization: the code before the kept-backing free. + 'old-reinit-guards': [(POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n')], + # Arm nothing while a reclaim's re-initialization is pending. + 'pending-reinit-arms-nothing': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n'), + # Arm the poll for the expired 50 ms instead of the rest of the long idle. + 'pending-delay-from-drain': (GROWTH, 'Some(min_idle_ns.saturating_sub(idle_ns))', + 'Some(DRAIN_IDLE_NS.saturating_sub(idle_ns))'), # Re-initialize at an idle with nothing grown (posted above the floor only). 'reclaim-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns', 'self.posted > floor && idle_ns >= min_idle_ns'), # Re-initialize at a client boundary with nothing grown (initial above half the maximum). 'drain-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS', 'self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS'), + # Arm the poll with nothing grown. + 'delay-nothing-grown': (GROWTH, ' let grown = self.posted > self.initial;\n', ' let grown = true;\n'), # Reset the posted count at the idle reclaim instead of at the next render's InitBM. 'reinit-at-idle-reclaim': (POOLS, ' if level >= 2 && idle {\n freed = self.free_backing();\n', ' if level >= 2 && idle {\n self.pools.ledger.reinit();\n freed = self.free_backing();\n'), @@ -259,12 +299,16 @@ 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), 'map-without-identity': ('freed_chunks_are_mapped_again', 'alias of another object'), - 'pending-reinit-stops-free': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), + 'pending-reinit-stops-free': ('client_boundary_reinit_arms_the_long_idle', 'poll woke in a loop'), + 'old-reinit-guards': ('client_boundary_reinit_arms_the_long_idle', 'kept backing not freed'), + 'pending-reinit-arms-nothing': ('client_boundary_reinit_arms_the_long_idle', 'kept backing not freed'), + 'pending-delay-from-drain': ('client_boundary_reinit_arms_the_long_idle', 'poll woke in a loop'), 'pending-free-at-level-1': ('client_boundary_reinit_at_level_1_keeps_the_backing', 'freed at level 1'), 'reclaim-pending-not-set': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), 'free-on-recovery-reinit': ('recovery_reinit_frees_nothing', 'freed for a recovery'), 'reclaim-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (64 blocks, floor 32)'), 'drain-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (3000 blocks'), + 'delay-nothing-grown': ('nothing_grown_arms_nothing', 'poll woke in a loop'), 'reinit-at-idle-reclaim': ('same_client_render_after_1_1s', 'posted count reset before the InitBM (1500)'), 'reinit-at-any-reclaim': ('same_client_render_after_60ms', 'posted count reset before the InitBM'), 'freed-without-walk': ('failed_vm_walk_counts_nothing_freed', 'freed counted with the VM walk failed'), @@ -1791,6 +1835,7 @@ def consts(text, names): check_at(&mut f, t0); check_at(&mut f, t0 + 50_000_000); assert!(f.recovery.reinit); + assert_eq!(f.reclaim_delay_ns(), None, "armed at level 1"); check_at(&mut f, t0 + 5_000_000_000); assert_eq!(f.pools.chunk_count(), chunks, "freed at level 1"); assert_eq!(f.pools.stats.freed, 0); @@ -1829,11 +1874,102 @@ def consts(text, names): f.recovery.reinit = true; let t0 = now(); check_at(&mut f, t0); + assert_eq!(f.reclaim_delay_ns(), None, "armed for a recovery's re-initialization"); check_at(&mut f, t0 + 5_000_000_000); assert_eq!(f.pools.chunk_count(), chunks, "freed for a recovery's re-initialization"); assert_eq!(f.pools.stats.reclaims, 0); } + /// The idle poll work as poll.rs drives it: a pass with nothing published notes the idle + /// and runs the reclaim check; an idle pass is re-armed only for reclaim_delay_ns(), + /// rounded up to whole ms, plus one 4 ms jiffy. Returns the wake times. + fn idle_polls(f: &mut Firmware, limit: usize) -> Vec { + let mut wakes = Vec::new(); + loop { + f.note_idle(); + f.reclaim_check(); + let Some(ns) = f.reclaim_delay_ns() else { + return wakes; + }; + assert!(wakes.len() < limit, "poll woke in a loop: {} wakes, next in {ns} ns", wakes.len()); + let t = now() + ns.div_ceil(1_000_000) * 1_000_000 + 4_000_000; + crate::NOW.with(|n| n.set(t)); + wakes.push(t); + } + } + + /// The same exit with the idle poll work: the poll is armed for the 50 ms + /// re-initialization, then for the rest of the long idle (not the expired 50 ms), where the + /// kept backing is freed; then nothing is armed. + #[test] + fn client_boundary_reinit_arms_the_long_idle() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(posted > 2047 && chunks > 2); + // The pass that retires the last flight notes the idle. + f.engine.flights.clear(); + f.note_idle(); + let t0 = now(); + take(); + let wakes = idle_polls(&mut f, 8); + let log = take(); + assert_eq!( + f.pools.chunk_count(), + 2, + "kept backing not freed: {} chunks, wakes {:?} ms", + f.pools.chunk_count(), + wakes.iter().map(|t| (t - t0) / 1_000_000).collect::>() + ); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!(f.pools.stats.reclaims, 1); + assert_eq!(f.pools.ledger.backed, 2 * CHUNK_BLOCKS); + assert_eq!(f.pools.stats.freed, (chunks - 2) as u64); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + // Two wakes: the re-initialization after 50 ms, the free after the long idle. + let ms: Vec = wakes.iter().map(|t| (t - t0) / 1_000_000).collect(); + assert_eq!(ms, vec![54, 1004], "wakes (ms after the idle began)"); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + // Idle much longer, or a job arriving: nothing more is due. + crate::NOW.with(|n| n.set(t0 + 100_000_000_000)); + f.reclaim_check(); + assert_eq!(f.reclaim_delay_ns(), None); + assert!(take().is_empty()); + } + + /// Nothing grown: the idle poll work arms nothing. + #[test] + fn nothing_grown_arms_nothing() { + for (blocks, floor) in [(64, 32), (3000, 1024)] { + defaults(); + crate::PARAM_BLOCKS.with(|b| b.set(blocks)); + crate::PARAM_RECLAIM_BLOCKS.with(|b| b.set(floor)); + let mut f = Firmware::new(); + f.engine.flights.clear(); + let wakes = idle_polls(&mut f, 8); + assert!(wakes.is_empty(), "armed with nothing grown ({blocks} blocks, floor {floor}): {} wakes", wakes.len()); + assert!(!f.recovery.reinit); + } + } + + /// An idle reclaim below half the maximum frees at the long idle; with its re-initialization + /// pending nothing more is armed. + #[test] + fn idle_reclaim_arms_nothing_after() { + let mut f = Firmware::new(); + grow_to(&mut f, 1500); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + f.note_idle(); + let t0 = now(); + let wakes = idle_polls(&mut f, 8); + let ms: Vec = wakes.iter().map(|t| (t - t0) / 1_000_000).collect(); + assert_eq!(ms, vec![1004]); + assert_eq!((f.pools.chunk_count(), f.pools.stats.freed), (2, (chunks - 2) as u64)); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + } + #[test] fn catch_up_refuses_flights_of_another_vm() { let mut f = Firmware::new(); @@ -1926,7 +2062,8 @@ def run(mutation): ' pub(crate) fn refill_backing(&mut self) {', ' pub(crate) fn note_idle(&mut self) {', ' pub(crate) fn reclaim_check(&mut self) {', - ' fn free_backing(&mut self) -> usize {')] + ' fn free_backing(&mut self) -> usize {', + ' pub(crate) fn reclaim_delay_ns(&self) -> Option {')] soft_max = item(pools, ' pub(crate) fn soft_max(&self) -> u32 {') pools_methods = (item(pools, ' fn map_heap(&self, vm: &VmState) -> Result {') + '\n' + item(pools, ' fn free_chunks(&mut self, keep: usize) -> usize {') + '\n' + From a089fd06488aa5edb09674a8d3ca71b56db07087 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:15:20 +0530 Subject: [PATCH 11/11] drm/asahi: g15: say when the boot loader left the GPU node disabled The boot loader enables the T6030/T6031 GPU node only when its chosen.asahi,t603x-gpu setting is on. With the setting off, as the installed m1n1 configuration of a J516C can have it (chosen.asahi,t6031-gpu=0), the node stays disabled. The adapter then never probes, and the kernel log gives no reason why there is no native G15 GPU. At module activation, log one line when /chosen carries asahi,t6031-gpu or asahi,t6030-gpu and the matching GPU node is disabled. The line names the node and the property's value. Nothing else changes; machines without the property log nothing. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik --- drivers/gpu/drm/asahi/g15/boot.rs | 1 + drivers/gpu/drm/asahi/g15/glue.c | 42 +++++++++++++++++++++++++++++ drivers/gpu/drm/asahi/g15/mod.rs | 2 ++ tools/asahi/g15-probe-init-tests.py | 2 ++ 4 files changed, 47 insertions(+) diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index e6278690d5f14a..cef9cc8b330059 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -63,6 +63,7 @@ pub(crate) mod glue { fn asahi_g15_release_power(dev: *mut core::ffi::c_void); fn asahi_g15_module_get(); pub(crate) fn asahi_g15_reprobe(); + pub(crate) fn asahi_g15_report_loader_gpu(); fn asahi_g15_adt_phram(start: *mut u64, size: *mut u64) -> core::ffi::c_int; fn asahi_g15_ram_disjoint(start: u64, size: u64) -> bool; fn asahi_g15_ram_within(start: u64, size: u64) -> bool; diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index 662c9c8258e213..5ceb20cd35d4b8 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -28,6 +28,7 @@ void asahi_g15_hold_power(struct device *dev); void asahi_g15_release_power(struct device *dev); void asahi_g15_module_get(void); void asahi_g15_reprobe(void); +void asahi_g15_report_loader_gpu(void); int asahi_g15_adt_phram(u64 *start, u64 *size); bool asahi_g15_ram_disjoint(u64 start, u64 size); bool asahi_g15_ram_within(u64 start, u64 size); @@ -39,6 +40,47 @@ int asahi_g15_pmp_get(struct device *dev); void asahi_g15_pmp_put(struct device *dev); bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); +/* + * The boot loader enables the T6030/T6031 GPU node only when its + * chosen.asahi,t603x-gpu setting is on; otherwise the node stays disabled, + * the adapter never probes and nothing says why native G15 is absent. Name + * the setting once, at module activation, on a machine whose /chosen has it + * and whose GPU node is disabled. Nothing else changes. + */ +void asahi_g15_report_loader_gpu(void) +{ + static const struct { + const char *prop; + const char *compatible; + } socs[] = { + { "asahi,t6031-gpu", "apple,agx-t6031" }, + { "asahi,t6030-gpu", "apple,agx-t6030" }, + }; + struct device_node *chosen, *np; + const char *value; + unsigned int i; + u32 cell; + + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return; + for (i = 0; i < ARRAY_SIZE(socs); i++) { + if (!of_property_present(chosen, socs[i].prop)) + continue; + np = of_find_compatible_node(NULL, NULL, socs[i].compatible); + if (np && !of_device_is_available(np)) { + if (!of_property_read_string(chosen, socs[i].prop, &value)) + pr_info("asahi: G15: native G15 not started: the boot loader left the GPU node %pOF disabled (/chosen/%s = \"%s\")\n", + np, socs[i].prop, value); + else if (!of_property_read_u32(chosen, socs[i].prop, &cell)) + pr_info("asahi: G15: native G15 not started: the boot loader left the GPU node %pOF disabled (/chosen/%s = %u)\n", + np, socs[i].prop, cell); + } + of_node_put(np); + } + of_node_put(chosen); +} + /* * The maintained m1n1 handoff keeps these four UAT regions in the linear map. * Require the exact enabled reserved-memory owner, in addition to the ADT diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 75d3ecc30ffe5a..faec77452ffc41 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -222,6 +222,8 @@ static MODULE_READY: AtomicBool = AtomicBool::new(false); /// No fallible module initialization may follow this call. pub(crate) fn activate() { MODULE_READY.store(true, Ordering::Release); + // SAFETY: The helper only reads the device tree. + unsafe { boot::glue::asahi_g15_report_loader_gpu() }; // SAFETY: Both pinned platform registrations are fully initialized and remain owned // by DriverModule. The C helper borrows their live driver pointer only for this call. unsafe { boot::glue::asahi_g15_reprobe() }; diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index 9acd1f012413ad..42b55f063359eb 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -174,6 +174,8 @@ def item(text, marker): mod boot { pub mod glue { pub fn retained() -> bool { crate::RETAINED.with(core::cell::Cell::get) } pub unsafe fn asahi_g15_reprobe() { crate::activation_reprobe(); } + // Reads the device tree and logs; no driver state. + pub unsafe fn asahi_g15_report_loader_gpu() {} }} '''