diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index b796645b471d80..98c95bf006a127 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -7,7 +7,7 @@ appledrm-y += afk.o dcp.o dcp-fabric.o dcp_backlight.o dptxep.o iomfb.o parser.o appledrm-y += dcp-hdmi.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o -appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o +appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o dcp-panel-edid.o appledrm-y += ibootep.o appledrm-y += dcpext_scanout.o appledrm-y += dcpext_drm.o @@ -25,6 +25,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_ready_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_dropped_swap_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp-panel-edid-test.o appledrm-y += iomfb_v26_6.o iomfb_v26_6_link.o j613_dcp_activate.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index d22b9a638169c9..651af197e08315 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -24,6 +24,7 @@ #include "dcp-announce.h" #include "dcp-display-metadata.h" #include "dcp-metadata-epoch.h" +#include "dcp-panel-edid.h" #include "dcp-180hz.h" #include "iomfb.h" #include "iomfb_h17p.h" @@ -174,6 +175,8 @@ struct dcp_panel { int height_mm; /// panel has a mini-LED backlight bool has_mini_led; + /// make and product for the base EDID, from the panel's compatible + struct dcp_panel_id id; }; enum dcp_iomfb_method_profile { diff --git a/drivers/gpu/drm/apple/dcp-panel-edid-test.c b/drivers/gpu/drm/apple/dcp-panel-edid-test.c new file mode 100644 index 00000000000000..399f6e5bfd6470 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-panel-edid-test.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +/* + * The built-in panel's EDID, read back with the DRM EDID helpers: the make, + * product, model year and native timing come from the panel's compatible and + * the model table; the base block holds the panel's geometry at 60 Hz with P3 + * colour and gamma 2.2, the DisplayID extension the 120 Hz timing; an update + * publishes both blocks on the connector and adds no mode. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "dcp-panel-edid.h" + +#define J516_W_MM 346 +#define J516_H_MM 216 + +static void panel_id_j516(struct kunit *test, struct dcp_panel_id *id) +{ + KUNIT_ASSERT_EQ(test, dcp_panel_id_from_compatible(id, "apple,panel-j516"), 0); +} + +static void panel_id_from_compatible(struct kunit *test) +{ + struct dcp_panel_id id; + + panel_id_j516(test, &id); + KUNIT_EXPECT_STREQ(test, id.vendor, "APP"); + KUNIT_EXPECT_STREQ(test, id.name, "J516"); + KUNIT_EXPECT_EQ(test, id.model_year, 2023); + KUNIT_EXPECT_EQ(test, id.timing.clock_khz, 1042130); + KUNIT_EXPECT_EQ(test, id.timing.vtotal, 2456); + /* A model without a known year and timing gets none, and no EDID. */ + KUNIT_ASSERT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel-j999"), 0); + KUNIT_EXPECT_EQ(test, id.model_year, 0); + KUNIT_EXPECT_EQ(test, id.timing.clock_khz, 0); +} + +static void panel_id_needs_a_model_and_a_known_vendor(struct kunit *test) +{ + struct dcp_panel_id id; + + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel-"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "panel-j516"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "acme,panel-j516"), -ENOENT); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, "apple,panel-j516 x"), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_id_from_compatible(&id, + "apple,panel-j5160000000000"), + -EINVAL); +} + +static const struct drm_edid *panel_edid_j516(struct kunit *test, + u8 e[DCP_PANEL_EDID_BLOCKS * EDID_LENGTH]) +{ + struct dcp_panel_id id; + const struct drm_edid *edid; + + panel_id_j516(test, &id); + KUNIT_ASSERT_EQ(test, dcp_panel_edid_build(&id, J516_W_MM, J516_H_MM, e), 0); + edid = drm_edid_alloc(e, DCP_PANEL_EDID_BLOCKS * EDID_LENGTH); + KUNIT_ASSERT_NOT_NULL(test, edid); + return edid; +} + +static u8 sum(const u8 *p, size_t len) +{ + u8 s = 0; + + while (len--) + s += *p++; + return s; +} + +static void panel_edid_parses_with_make_product_and_size(struct kunit *test) +{ + struct drm_edid_product_id product; + u8 e[DCP_PANEL_EDID_BLOCKS * EDID_LENGTH]; + const struct drm_edid *edid = panel_edid_j516(test, e); + const struct edid *raw; + char vendor[4], name[14]; + + KUNIT_EXPECT_TRUE(test, drm_edid_valid(edid)); + drm_edid_get_product_id(edid, &product); + drm_edid_decode_mfg_id(be16_to_cpu(product.manufacturer_name), vendor); + KUNIT_EXPECT_STREQ(test, vendor, "APP"); + KUNIT_EXPECT_EQ(test, le16_to_cpu(product.product_code), 0); + KUNIT_EXPECT_EQ(test, le32_to_cpu(product.serial_number), 0); + + raw = drm_edid_raw(edid); + KUNIT_ASSERT_NOT_NULL(test, raw); + drm_edid_get_monitor_name(raw, name, sizeof(name)); + KUNIT_EXPECT_STREQ(test, name, "J516"); + /* The connector's size (the firmware's), in whole centimetres. */ + KUNIT_EXPECT_EQ(test, raw->width_cm, 35); + KUNIT_EXPECT_EQ(test, raw->height_cm, 22); + /* Gamma 2.2, not 1.00 (0) and not "in an extension" (0xff). */ + KUNIT_EXPECT_EQ(test, raw->gamma, 0x78); + /* Model year 2023: week 0xff, year 2023 - 1990. */ + KUNIT_EXPECT_EQ(test, raw->mfg_week, 0xff); + KUNIT_EXPECT_EQ(test, raw->mfg_year, 33); + /* P3 red (0.680, 0.320) and the D65 white point (0.3127, 0.3290), 10-bit. */ + KUNIT_EXPECT_EQ(test, raw->red_x << 2 | raw->red_green_lo >> 6, 696); + KUNIT_EXPECT_EQ(test, raw->red_y << 2 | (raw->red_green_lo >> 4 & 3), 328); + KUNIT_EXPECT_EQ(test, raw->white_x << 2 | (raw->blue_white_lo >> 2 & 3), 320); + KUNIT_EXPECT_EQ(test, raw->white_y << 2 | (raw->blue_white_lo & 3), 337); + /* One extension; both blocks sum to 0. */ + KUNIT_EXPECT_EQ(test, raw->extensions, 1); + KUNIT_EXPECT_EQ(test, sum(e, EDID_LENGTH), 0); + KUNIT_EXPECT_EQ(test, sum(e + EDID_LENGTH, EDID_LENGTH), 0); + drm_edid_free(edid); +} + +/* + * The base block's first descriptor is the panel's geometry at 60 Hz (its 120 Hz + * clock does not fit the 16-bit field); the DisplayID extension holds the real + * 120 Hz timing as its preferred Type VII timing. + */ +static void panel_edid_timings_are_the_panels(struct kunit *test) +{ + u8 e[DCP_PANEL_EDID_BLOCKS * EDID_LENGTH]; + const struct drm_edid *edid = panel_edid_j516(test, e); + const struct edid *raw = drm_edid_raw(edid); + const struct detailed_timing *d = &raw->detailed_timings[0]; + const struct detailed_pixel_timing *pt = &d->data.pixel_data; + const u8 *ext = e + EDID_LENGTH, *section = ext + 1, *t; + unsigned int i, names = 0; + + /* 3536 x 2456 x 60 Hz = 521.06 MHz, in 10 kHz. */ + KUNIT_EXPECT_EQ(test, le16_to_cpu(d->pixel_clock), 52106); + KUNIT_EXPECT_EQ(test, pt->hactive_lo | (pt->hactive_hblank_hi >> 4) << 8, 3456); + KUNIT_EXPECT_EQ(test, pt->hblank_lo | (pt->hactive_hblank_hi & 0xf) << 8, 80); + KUNIT_EXPECT_EQ(test, pt->vactive_lo | (pt->vactive_vblank_hi >> 4) << 8, 2234); + KUNIT_EXPECT_EQ(test, pt->vblank_lo | (pt->vactive_vblank_hi & 0xf) << 8, 222); + KUNIT_EXPECT_EQ(test, pt->hsync_offset_lo, 8); + KUNIT_EXPECT_EQ(test, pt->hsync_pulse_width_lo, 32); + /* The front porch at the field's 63, the sync width the panel's 8. */ + KUNIT_EXPECT_EQ(test, (pt->vsync_offset_pulse_width_lo >> 4) | + (pt->hsync_vsync_offset_pulse_width_hi >> 2 & 3) << 4, 63); + KUNIT_EXPECT_EQ(test, pt->vsync_offset_pulse_width_lo & 0xf, 8); + KUNIT_EXPECT_EQ(test, pt->width_mm_lo | (pt->width_height_mm_hi >> 4) << 8, J516_W_MM); + KUNIT_EXPECT_EQ(test, pt->height_mm_lo | (pt->width_height_mm_hi & 0xf) << 8, J516_H_MM); + /* It is not the preferred mode with its refresh rate. */ + KUNIT_EXPECT_EQ(test, raw->features & DRM_EDID_FEATURE_PREFERRED_TIMING, 0); + for (i = 1; i < ARRAY_SIZE(raw->detailed_timings); i++) { + const struct detailed_timing *o = &raw->detailed_timings[i]; + + KUNIT_EXPECT_EQ(test, le16_to_cpu(o->pixel_clock), 0); + KUNIT_EXPECT_NE(test, o->data.other_data.type, EDID_DETAIL_MONITOR_SERIAL); + KUNIT_EXPECT_NE(test, o->data.other_data.type, EDID_DETAIL_MONITOR_RANGE); + names += o->data.other_data.type == EDID_DETAIL_MONITOR_NAME; + } + KUNIT_EXPECT_EQ(test, names, 1); + for (i = 0; i < sizeof(raw->established_timings); i++) + KUNIT_EXPECT_EQ(test, ((const u8 *)&raw->established_timings)[i], 0); + + /* DisplayID 2.0, generic display, Display Parameters then one Type VII timing. */ + KUNIT_EXPECT_EQ(test, ext[0], 0x70); + KUNIT_EXPECT_EQ(test, section[0], 0x20); + KUNIT_EXPECT_EQ(test, section[2], 0x02); + KUNIT_ASSERT_EQ(test, section[1], 3 + 29 + 3 + 20); + KUNIT_EXPECT_EQ(test, sum(section, 4 + section[1] + 1), 0); + KUNIT_EXPECT_EQ(test, section[4], 0x21); + KUNIT_EXPECT_EQ(test, section[6], 29); + /* Image size in 0.1 mm, the native pixels, gamma 2.2. */ + KUNIT_EXPECT_EQ(test, section[7] | section[8] << 8, J516_W_MM * 10); + KUNIT_EXPECT_EQ(test, section[9] | section[10] << 8, J516_H_MM * 10); + KUNIT_EXPECT_EQ(test, section[11] | section[12] << 8, 3456); + KUNIT_EXPECT_EQ(test, section[13] | section[14] << 8, 2234); + KUNIT_EXPECT_EQ(test, section[7 + 28], 0x78); + t = section + 4 + 3 + 29; + KUNIT_EXPECT_EQ(test, t[0], 0x22); + KUNIT_EXPECT_EQ(test, t[2], 20); + t += 3; + /* 1042130 kHz, preferred, and the DCP mode line's porches (each minus one). */ + KUNIT_EXPECT_EQ(test, t[0] | t[1] << 8 | t[2] << 16, 1042130 - 1); + KUNIT_EXPECT_EQ(test, t[3] & 0x80, 0x80); + KUNIT_EXPECT_EQ(test, t[4] | t[5] << 8, 3456 - 1); + KUNIT_EXPECT_EQ(test, t[6] | t[7] << 8, 3536 - 3456 - 1); + KUNIT_EXPECT_EQ(test, (t[8] | t[9] << 8) & 0x7fff, 3464 - 3456 - 1); + KUNIT_EXPECT_EQ(test, t[10] | t[11] << 8, 3496 - 3464 - 1); + KUNIT_EXPECT_EQ(test, t[12] | t[13] << 8, 2234 - 1); + KUNIT_EXPECT_EQ(test, t[14] | t[15] << 8, 2456 - 2234 - 1); + KUNIT_EXPECT_EQ(test, (t[16] | t[17] << 8) & 0x7fff, 2352 - 2234 - 1); + KUNIT_EXPECT_EQ(test, t[18] | t[19] << 8, 2360 - 2352 - 1); + drm_edid_free(edid); +} + +static void panel_edid_refuses_a_size_it_cannot_hold(struct kunit *test) +{ + u8 e[DCP_PANEL_EDID_BLOCKS * EDID_LENGTH]; + struct dcp_panel_id id; + + panel_id_j516(test, &id); + KUNIT_EXPECT_EQ(test, dcp_panel_edid_build(&id, 0, J516_H_MM, e), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_panel_edid_build(&id, J516_W_MM, 2560, e), -EINVAL); + id.model_year = 2005; + KUNIT_EXPECT_EQ(test, dcp_panel_edid_build(&id, J516_W_MM, J516_H_MM, e), -EINVAL); + id.model_year = 2023; + id.timing.clock_khz = 0; + KUNIT_EXPECT_EQ(test, dcp_panel_edid_build(&id, J516_W_MM, J516_H_MM, e), -EINVAL); + panel_id_j516(test, &id); + id.vendor[0] = '\0'; + KUNIT_EXPECT_EQ(test, dcp_panel_edid_build(&id, J516_W_MM, J516_H_MM, e), -EINVAL); +} + +struct panel_edid_test { + struct drm_device drm; + struct drm_connector connector; +}; + +/* The DRM KUnit device is atomic: a connector needs its state helpers. */ +static const struct drm_connector_funcs panel_edid_connector_funcs = { + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static void panel_edid_update_publishes_the_block(struct kunit *test) +{ + struct panel_edid_test *t; + struct drm_connector *connector; + struct dcp_panel_id id; + struct device *dev; + int ret; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + t = drm_kunit_helper_alloc_drm_device(test, dev, struct panel_edid_test, drm, + DRIVER_MODESET); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t); + connector = &t->connector; + ret = drmm_connector_init(&t->drm, connector, &panel_edid_connector_funcs, + DRM_MODE_CONNECTOR_eDP, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + + panel_id_j516(test, &id); + mutex_lock(&t->drm.mode_config.mutex); + ret = dcp_panel_edid_update(connector, &id, J516_W_MM, J516_H_MM); + mutex_unlock(&t->drm.mode_config.mutex); + KUNIT_ASSERT_EQ(test, ret, 0); + + /* The update sets 350x220 from the block; dcp_get_modes() restores the mm. */ + KUNIT_EXPECT_EQ(test, connector->display_info.width_mm, 350); + KUNIT_EXPECT_EQ(test, connector->display_info.height_mm, 220); + KUNIT_ASSERT_NOT_NULL(test, connector->edid_blob_ptr); + KUNIT_EXPECT_EQ(test, connector->edid_blob_ptr->length, 2 * EDID_LENGTH); + /* The update adds no mode: the modes stay the firmware's. */ + KUNIT_EXPECT_TRUE(test, list_empty(&connector->probed_modes)); +} + +static struct kunit_case panel_edid_cases[] = { + KUNIT_CASE(panel_id_from_compatible), + KUNIT_CASE(panel_id_needs_a_model_and_a_known_vendor), + KUNIT_CASE(panel_edid_parses_with_make_product_and_size), + KUNIT_CASE(panel_edid_timings_are_the_panels), + KUNIT_CASE(panel_edid_refuses_a_size_it_cannot_hold), + KUNIT_CASE(panel_edid_update_publishes_the_block), + {} +}; + +static struct kunit_suite panel_edid_suite = { + .name = "apple-dcp-panel-edid", + .test_cases = panel_edid_cases, +}; + +kunit_test_suite(panel_edid_suite); diff --git a/drivers/gpu/drm/apple/dcp-panel-edid.c b/drivers/gpu/drm/apple/dcp-panel-edid.c new file mode 100644 index 00000000000000..97eee161016978 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-panel-edid.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * An EDID for a built-in panel. + * + * The DCP gives no EDID for a built-in panel, so a compositor sees no make, + * model or physical size for it. This builds one from what the panel's + * device-tree node and a table of the known panel models say: the vendor and + * model of its most specific compatible ("apple,panel-j516": make APP, name + * J516), the size the connector has, the model year, the panel's native timing + * as the DCP's mode line gives it with all rows shown, and the colour of a P3 + * wide-colour panel (P3 primaries, D65 white point and a nominal gamma of + * 2.2, which the sRGB transfer function of Display P3 approximates). + * + * The native timing is 3456x2234 at 120 Hz with a pixel clock of 1042 MHz. + * The base block's detailed timing cannot hold it: its pixel clock field is + * 16 bits in 10 kHz units (at most 655.35 MHz) and its vertical front porch 6 + * bits. The base block therefore carries the panel's geometry at 60 Hz (the + * same totals and horizontal porches, the vertical front porch at most 63 + * lines), and a DisplayID 2.0 extension carries the real 120 Hz timing as its + * preferred timing, with the display parameters. Neither is a mode source: + * the driver publishes the firmware's modes and never adds modes from this + * EDID. It makes no other claim: no product code or serial number, no + * manufacture week, no luminance and no refresh range. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "dcp-panel-edid.h" + +/* Device-tree vendor prefixes and their PNP IDs. */ +static const struct { + const char *prefix; + const char *pnp; +} dcp_panel_vendors[] = { + { "apple", "APP" }, +}; + +/* + * Panel models: the year their machines came out (the EDID model year) and the + * native timing of the DCP's mode line, with the rows the notch hides shown. + */ +static const struct { + const char *name; + u16 year; + struct dcp_panel_timing timing; +} dcp_panel_models[] = { + /* MacBook Pro (16-inch, M3 Pro or M3 Max): 3456x2234 at 120 Hz */ + { "J516", 2023, { 1042130, 3456, 3464, 3496, 3536, 2234, 2352, 2360, 2456 } }, +}; + +/* EDID 1.4: a nominal gamma of 2.2 is (2.2 - 1) * 100 = 120. */ +#define DCP_PANEL_EDID_GAMMA 120 +/* EDID 1.4: week 0xff marks the year byte as the model year, from 1990. */ +#define DCP_PANEL_EDID_MODEL_YEAR 0xff +/* The base block's detailed timing: the panel's geometry at this rate. */ +#define DCP_PANEL_EDID_DTD_HZ 60 +/* EDID 1.4 detailed timing: the vertical front porch field is 6 bits. */ +#define DCP_PANEL_EDID_DTD_VFP_MAX 63 + +/* P3 primaries and the D65 white point, CIE 1931 x and y in 1/10000. */ +static const u16 dcp_panel_p3[8] = { 6800, 3200, 2650, 6900, 1500, 600, 3127, 3290 }; + +/* DisplayID 2.0 (VESA DisplayID Standard v2.0). */ +#define DISPLAYID_EXT 0x70 /* EDID extension tag */ +#define DISPLAYID_V20 0x20 /* structure version 2, revision 0 */ +#define DISPLAYID_GENERIC 0x02 /* primary use case: generic display */ +#define DISPLAYID_PARAMS 0x21 /* Display Parameters data block */ +#define DISPLAYID_PARAMS_LEN 29 +#define DISPLAYID_TYPE7 0x22 /* Type VII detailed timing data block */ +#define DISPLAYID_TYPE7_LEN 20 +#define DISPLAYID_PREFERRED 0x80 /* timing options: preferred */ +#define DISPLAYID_ASPECT_NONE 0x08 /* aspect ratio not given (from the pixels) */ +#define DISPLAYID_SYNC_POS 0x8000 /* front porch word: positive sync */ +#define DISPLAYID_NO_SPEAKERS 0x80 /* features: audio not integrated */ +#define DISPLAYID_TECH_LCD 0x10 /* device technology: active-matrix LCD */ + +int dcp_panel_id_from_compatible(struct dcp_panel_id *id, const char *compatible) +{ + const char *comma = strchr(compatible, ','), *model; + size_t vendor_len, len, i; + + memset(id, 0, sizeof(*id)); + if (!comma) + return -EINVAL; + vendor_len = comma - compatible; + model = comma + 1; + if (str_has_prefix(model, "panel-")) + model += strlen("panel-"); + else + return -EINVAL; + len = strlen(model); + if (!len || len > DCP_PANEL_NAME_LEN) + return -EINVAL; + for (i = 0; i < len; i++) { + if (!isalnum(model[i]) && model[i] != '-') + return -EINVAL; + id->name[i] = toupper(model[i]); + } + + for (i = 0; i < ARRAY_SIZE(dcp_panel_models); i++) { + if (!strcmp(id->name, dcp_panel_models[i].name)) { + id->model_year = dcp_panel_models[i].year; + id->timing = dcp_panel_models[i].timing; + } + } + for (i = 0; i < ARRAY_SIZE(dcp_panel_vendors); i++) { + if (strlen(dcp_panel_vendors[i].prefix) == vendor_len && + !strncmp(compatible, dcp_panel_vendors[i].prefix, vendor_len)) { + strscpy(id->vendor, dcp_panel_vendors[i].pnp); + return 0; + } + } + memset(id, 0, sizeof(*id)); + return -ENOENT; +} + +static void put_le16(u8 *p, u16 v) +{ + p[0] = v; + p[1] = v >> 8; +} + +static void checksum(u8 *start, size_t len) +{ + u8 sum = 0; + size_t i; + + for (i = 0; i < len - 1; i++) + sum += start[i]; + start[len - 1] = -sum; +} + +static bool timing_valid(const struct dcp_panel_timing *t) +{ + return t->clock_khz && t->hdisplay < t->hsync_start && t->hsync_start < t->hsync_end && + t->hsync_end < t->htotal && t->htotal - t->hdisplay < 4096 && + t->vdisplay < t->vsync_start && t->vsync_start < t->vsync_end && + t->vsync_end < t->vtotal && t->vtotal - t->vdisplay < 4096 && + t->hsync_start - t->hdisplay < 1024 && t->hsync_end - t->hsync_start < 1024 && + t->vsync_end - t->vsync_start < 64; +} + +/* The base block's detailed timing: the panel's geometry at DCP_PANEL_EDID_DTD_HZ. */ +static void base_dtd(u8 *d, const struct dcp_panel_timing *t, int width_mm, int height_mm) +{ + u32 clock = DIV_ROUND_CLOSEST((u32)t->htotal * t->vtotal * DCP_PANEL_EDID_DTD_HZ, 10000); + u16 hblank = t->htotal - t->hdisplay, vblank = t->vtotal - t->vdisplay; + u16 hfp = t->hsync_start - t->hdisplay, hsw = t->hsync_end - t->hsync_start; + u16 vfp = min_t(u16, t->vsync_start - t->vdisplay, DCP_PANEL_EDID_DTD_VFP_MAX); + u16 vsw = t->vsync_end - t->vsync_start; + + put_le16(d, clock); + d[2] = t->hdisplay; + d[3] = hblank; + d[4] = (t->hdisplay >> 8) << 4 | hblank >> 8; + d[5] = t->vdisplay; + d[6] = vblank; + d[7] = (t->vdisplay >> 8) << 4 | vblank >> 8; + d[8] = hfp; + d[9] = hsw; + d[10] = (vfp & 0xf) << 4 | (vsw & 0xf); + d[11] = (hfp >> 8) << 6 | (hsw >> 8) << 4 | (vfp >> 4) << 2 | (vsw >> 4); + d[12] = width_mm; + d[13] = height_mm; + d[14] = (width_mm >> 8) << 4 | height_mm >> 8; + /* Digital separate sync, positive vertical and horizontal sync. */ + d[17] = 0x1e; +} + +/* The P3/D65 chromaticity of the base block, in 10-bit fractions. */ +static void base_chromaticity(struct edid *block) +{ + u16 v[8]; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(v); i++) + v[i] = DIV_ROUND_CLOSEST(dcp_panel_p3[i] * 1024, 10000); + block->red_green_lo = (v[0] & 3) << 6 | (v[1] & 3) << 4 | (v[2] & 3) << 2 | (v[3] & 3); + block->blue_white_lo = (v[4] & 3) << 6 | (v[5] & 3) << 4 | (v[6] & 3) << 2 | (v[7] & 3); + block->red_x = v[0] >> 2; + block->red_y = v[1] >> 2; + block->green_x = v[2] >> 2; + block->green_y = v[3] >> 2; + block->blue_x = v[4] >> 2; + block->blue_y = v[5] >> 2; + block->white_x = v[6] >> 2; + block->white_y = v[7] >> 2; +} + +static int base_block(const struct dcp_panel_id *id, int width_mm, int height_mm, + struct edid *block) +{ + struct detailed_non_pixel *name; + u16 manufacturer = 0; + size_t len = strnlen(id->name, sizeof(id->name)); + unsigned int i; + + for (i = 0; i < 3; i++) { + if (id->vendor[i] < 'A' || id->vendor[i] > 'Z') + return -EINVAL; + manufacturer = manufacturer << 5 | (id->vendor[i] - '@'); + } + + memset(block, 0, sizeof(*block)); + memcpy(block->header, "\x00\xff\xff\xff\xff\xff\xff\x00", sizeof(block->header)); + block->product_id.manufacturer_name = cpu_to_be16(manufacturer); + block->mfg_week = DCP_PANEL_EDID_MODEL_YEAR; + block->mfg_year = id->model_year - 1990; + block->version = 1; + block->revision = 4; + block->input = DRM_EDID_INPUT_DIGITAL; + block->width_cm = DIV_ROUND_CLOSEST(width_mm, 10); + block->height_cm = DIV_ROUND_CLOSEST(height_mm, 10); + block->gamma = DCP_PANEL_EDID_GAMMA; + base_chromaticity(block); + /* Standard timings 0x0101 are unused slots. */ + memset(block->standard_timings, 0x01, sizeof(block->standard_timings)); + + /* EDID 1.4: the first descriptor is a detailed timing; the second names the panel. */ + base_dtd((u8 *)&block->detailed_timings[0], &id->timing, width_mm, height_mm); + name = &block->detailed_timings[1].data.other_data; + name->type = EDID_DETAIL_MONITOR_NAME; + memset(name->data.str.str, ' ', sizeof(name->data.str.str)); + memcpy(name->data.str.str, id->name, len); + if (len < sizeof(name->data.str.str)) + name->data.str.str[len] = '\n'; + for (i = 2; i < ARRAY_SIZE(block->detailed_timings); i++) + block->detailed_timings[i].data.other_data.type = 0x10; + block->extensions = DCP_PANEL_EDID_BLOCKS - 1; + checksum((u8 *)block, EDID_LENGTH); + return 0; +} + +/* DisplayID 2.0 CIE 1931 x/y pair in 12-bit fractions, packed in 3 bytes. */ +static void displayid_xy(u8 *p, u16 x, u16 y) +{ + u16 X = DIV_ROUND_CLOSEST(x * 4095, 10000), Y = DIV_ROUND_CLOSEST(y * 4095, 10000); + + p[0] = X; + p[1] = (X >> 8) | (Y & 0xf) << 4; + p[2] = Y >> 4; +} + +/* The DisplayID 2.0 extension: display parameters and the 120 Hz Type VII timing. */ +static void displayid_block(const struct dcp_panel_id *id, int width_mm, int height_mm, + u8 *ext) +{ + const struct dcp_panel_timing *t = &id->timing; + u8 *section = ext + 1, *p, *b; + unsigned int i; + + memset(ext, 0, EDID_LENGTH); + ext[0] = DISPLAYID_EXT; + section[0] = DISPLAYID_V20; + section[1] = 3 + DISPLAYID_PARAMS_LEN + 3 + DISPLAYID_TYPE7_LEN; + section[2] = DISPLAYID_GENERIC; + section[3] = 0; /* no further DisplayID extensions */ + + b = section + 4; + b[0] = DISPLAYID_PARAMS; + b[2] = DISPLAYID_PARAMS_LEN; + p = b + 3; + put_le16(p + 0, width_mm * 10); /* 0.1 mm */ + put_le16(p + 2, height_mm * 10); + put_le16(p + 4, t->hdisplay); + put_le16(p + 6, t->vdisplay); + /* Left to right, top to bottom; CIE 1931; the speakers are not the panel's. */ + p[8] = DISPLAYID_NO_SPEAKERS; + for (i = 0; i < 4; i++) + displayid_xy(p + 9 + 3 * i, dcp_panel_p3[2 * i], dcp_panel_p3[2 * i + 1]); + /* Bytes 21-26, the luminances, are not known: left 0. Colour depth not given. */ + p[27] = DISPLAYID_TECH_LCD; + p[28] = DCP_PANEL_EDID_GAMMA; + + b = p + DISPLAYID_PARAMS_LEN; + b[0] = DISPLAYID_TYPE7; + b[2] = DISPLAYID_TYPE7_LEN; + p = b + 3; + /* Pixel clock in kHz minus one, 24 bits. */ + p[0] = (t->clock_khz - 1); + p[1] = (t->clock_khz - 1) >> 8; + p[2] = (t->clock_khz - 1) >> 16; + p[3] = DISPLAYID_PREFERRED | DISPLAYID_ASPECT_NONE; + /* Each field minus one; the front porch words carry the sync polarity. */ + put_le16(p + 4, t->hdisplay - 1); + put_le16(p + 6, t->htotal - t->hdisplay - 1); + put_le16(p + 8, (t->hsync_start - t->hdisplay - 1) | DISPLAYID_SYNC_POS); + put_le16(p + 10, t->hsync_end - t->hsync_start - 1); + put_le16(p + 12, t->vdisplay - 1); + put_le16(p + 14, t->vtotal - t->vdisplay - 1); + put_le16(p + 16, (t->vsync_start - t->vdisplay - 1) | DISPLAYID_SYNC_POS); + put_le16(p + 18, t->vsync_end - t->vsync_start - 1); + + /* The section's checksum after its payload, then the block's. */ + checksum(section, 4 + section[1] + 1); + checksum(ext, EDID_LENGTH); +} + +int dcp_panel_edid_build(const struct dcp_panel_id *id, int width_mm, int height_mm, + u8 edid[DCP_PANEL_EDID_BLOCKS * EDID_LENGTH]) +{ + size_t len = strnlen(id->name, sizeof(id->name)); + int ret; + + if (width_mm <= 0 || height_mm <= 0 || width_mm > 4095 || height_mm > 4095 || + DIV_ROUND_CLOSEST(width_mm, 10) > 255 || DIV_ROUND_CLOSEST(height_mm, 10) > 255) + return -EINVAL; + if (strnlen(id->vendor, sizeof(id->vendor)) != 3 || !len || len > DCP_PANEL_NAME_LEN) + return -EINVAL; + /* EDID 1.4 reserves the years before 2006. */ + if (id->model_year < 2006 || id->model_year > 1990 + 255) + return -EINVAL; + if (!timing_valid(&id->timing)) + return -EINVAL; + ret = base_block(id, width_mm, height_mm, (struct edid *)edid); + if (ret) + return ret; + displayid_block(id, width_mm, height_mm, edid + EDID_LENGTH); + return 0; +} + +int dcp_panel_edid_update(struct drm_connector *connector, const struct dcp_panel_id *id, + int width_mm, int height_mm) +{ + const struct drm_edid *edid; + u8 *blocks; + int ret; + + blocks = kmalloc(DCP_PANEL_EDID_BLOCKS * EDID_LENGTH, GFP_KERNEL); + if (!blocks) + return -ENOMEM; + ret = dcp_panel_edid_build(id, width_mm, height_mm, blocks); + if (ret) + goto out; + edid = drm_edid_alloc(blocks, DCP_PANEL_EDID_BLOCKS * EDID_LENGTH); + if (!edid) { + ret = -ENOMEM; + goto out; + } + /* This sets the connector's size from the block, in whole centimetres. */ + ret = drm_edid_connector_update(connector, edid); + drm_edid_free(edid); +out: + kfree(blocks); + return ret; +} diff --git a/drivers/gpu/drm/apple/dcp-panel-edid.h b/drivers/gpu/drm/apple/dcp-panel-edid.h new file mode 100644 index 00000000000000..1e9a6242831b11 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-panel-edid.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* A base EDID for a built-in panel, from its device-tree node. */ +#ifndef __APPLE_DCP_PANEL_EDID_H__ +#define __APPLE_DCP_PANEL_EDID_H__ + +#include + +/* The monitor-name text of an EDID descriptor holds 13 characters. */ +#define DCP_PANEL_NAME_LEN 13 + +/* An EDID of a base block and one DisplayID extension. */ +#define DCP_PANEL_EDID_BLOCKS 2 + +/* A panel's native timing, as the DCP's mode line gives it (all rows shown). */ +struct dcp_panel_timing { + u32 clock_khz; + u16 hdisplay, hsync_start, hsync_end, htotal; + u16 vdisplay, vsync_start, vsync_end, vtotal; +}; + +/* The make, product, model year and native timing of a panel. */ +struct dcp_panel_id { + char vendor[4]; /* PNP ID, empty if unknown */ + char name[DCP_PANEL_NAME_LEN + 1]; /* monitor name */ + u16 model_year; /* the model's year, 0 if unknown */ + struct dcp_panel_timing timing; /* zero if unknown */ +}; + +struct drm_connector; + +int dcp_panel_id_from_compatible(struct dcp_panel_id *id, const char *compatible); +int dcp_panel_edid_build(const struct dcp_panel_id *id, int width_mm, int height_mm, + u8 edid[DCP_PANEL_EDID_BLOCKS * 128]); +int dcp_panel_edid_update(struct drm_connector *connector, const struct dcp_panel_id *id, + int width_mm, int height_mm); + +#endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8c877d0c2434b9..835b7293823edc 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3768,6 +3768,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat if (panel_np) { const char height_prop[2][16] = { "adj-height-mm", "height-mm" }; + const char *compatible; if (of_device_is_available(panel_np)) { ret = of_property_read_u32(panel_np, "apple,max-brightness", @@ -3780,6 +3781,11 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat /* use adjusted height as long as the notch is hidden */ of_property_read_u32(panel_np, height_prop[!dcp->notch_height], &dcp->panel.height_mm); + /* The most specific compatible names the panel for its EDID. */ + if (iomfb_v14_7_base_edid(dcp) && + !of_property_read_string_index(panel_np, "compatible", 0, &compatible) && + dcp_panel_id_from_compatible(&dcp->panel.id, compatible)) + dev_dbg(dev, "no EDID make and product in '%s'\n", compatible); of_node_put(panel_np); dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1efc58efd6f620..e505d0788388d0 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1878,6 +1878,22 @@ int dcp_get_modes(struct drm_connector *connector) } else { apple_connector->drm_edid_firmware_derived = false; } + } else if (nr_modes && dcp_is_v14_7_panel(dcp) && dcp->panel.id.vendor[0]) { + /* + * No EDID comes for this panel: publish the base block of its + * node, sized as the connector is (the firmware's size, else the + * node's), then give the connector back its size in millimetres. + */ + int update; + + dcp_update_connector_dimensions(dcp, connector); + update = dcp_panel_edid_update(connector, &dcp->panel.id, + connector->display_info.width_mm, + connector->display_info.height_mm); + dcp_update_connector_dimensions(dcp, connector); + apple_connector->drm_edid_firmware_derived = false; + if (update) + dev_warn_once(dcp->dev, "panel EDID publication failed: %d\n", update); } else { u64 generation = 0; const struct drm_edid *derived = nr_modes ? diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 474841ff8d0aec..0acafac676dcdd 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -159,6 +159,8 @@ static const struct dcp_v14_board dcp_v14_board_j516c = { .panel_height = 2234, .promotion = true, .log_bringup = true, + .base_edid = true, + .swap_timestamps = true, }; /* @@ -1541,6 +1543,16 @@ const char *iomfb_v14_7_board_name(const struct dcp_v14_board *board) return board->name; } +bool dcp_v14_board_stamps_swaps(const struct dcp_v14_board *board, bool external) +{ + return board && !external && board->swap_timestamps; +} + +bool iomfb_v14_7_base_edid(const struct apple_dcp *dcp) +{ + return dcp->v14 && dcp->v14->board && dcp->v14->board->base_edid; +} + int iomfb_v14_7_probe(struct apple_dcp *dcp) { struct device *dev = dcp->dev; @@ -2609,6 +2621,8 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, if (!ret) { dcp_v14_encode_swap(swap, id, DCP_V14_BLACK, surface, iova, width, height, dst_y, DCP_V14_LAYER); + if (dcp_v14_board_stamps_swaps(v14->board, v14->external)) + dcp_v14_swap_timestamps(swap); ret = dcp_v14_call_guarded(v14, A(407), swap, DCP_V14_SWAP_SIZE, result, sizeof(result), id, dcp_v14_swap_admit, &admission); /* The swap status sits at byte 5 of the reply. */ diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index f9bef6ed0f109f..be801416508af1 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -28,6 +28,8 @@ const __be32 *of_translate_dma_region_checked(struct device_node *dev, */ const struct dcp_v14_board *iomfb_v14_7_board(struct device *dev); const char *iomfb_v14_7_board_name(const struct dcp_v14_board *board); +/* Whether the internal panel's board record gives it a base EDID. */ +bool iomfb_v14_7_base_edid(const struct apple_dcp *dcp); /* Platform probe: firmware identity, boot loader handoff, PMP acknowledgment. */ int iomfb_v14_7_probe(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_board.h b/drivers/gpu/drm/apple/iomfb_v14_7_board.h index 368dab49c29445..c677bbbafa04db 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_board.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7_board.h @@ -39,6 +39,18 @@ struct dcp_v14_board { * releases, so they are only set where tested. */ u32 ctm_set, ctm_get; + /* + * The panel gets a base EDID (make, model and size) built from its + * device-tree node, as the firmware gives none. + */ + bool base_edid; + /* + * The panel's swaps carry the constant timestamp + * DCP_V14_SWAP_TIMESTAMP_VALUE in slots 0, 1 and 6. With zero + * timestamps this board's firmware completes at most one swap per + * 16.7 ms, so a 120 Hz panel presents at 60 fps. + */ + bool swap_timestamps; }; /* Whether the DCP node or the machine (as @ctx gives them) has @compat. */ @@ -59,4 +71,11 @@ const struct dcp_v14_board *dcp_v14_board_select(dcp_v14_compat_fn dcp_is, */ bool dcp_v14_board_admits(const struct dcp_v14_board *board, const char *uuid); +/* + * Whether a swap carries timestamps: only the internal panel of a board whose + * record asks for them. External processors (@external, no record) and every + * other board send zero timestamps. + */ +bool dcp_v14_board_stamps_swaps(const struct dcp_v14_board *board, bool external); + #endif /* __APPLE_IOMFB_V14_7_BOARD_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_swap.h b/drivers/gpu/drm/apple/iomfb_v14_7_swap.h index aab3351dd9eea4..7fe32cabfae65c 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_swap.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7_swap.h @@ -11,6 +11,8 @@ #define DCP_V14_SURFACE_SIZE 0x22c /* Offsets in the swap record. Rectangles are x, y, w, h (u32 each). */ +#define DCP_V14_SWAP_TIMESTAMP(n) ((n) * 8) /* u64, slots 0 to 6 */ +#define DCP_V14_SWAP_TIMESTAMPS 7 #define DCP_V14_SWAP_ID 0x98 #define DCP_V14_SWAP_SURF_ID(p) (0x9c + (p) * 4) #define DCP_V14_SWAP_SRC(p) (0xac + (p) * 16) /* in the surface */ @@ -115,4 +117,19 @@ static inline void dcp_v14_encode_swap(u8 *swap, u32 id, u32 background, const u } } +/* + * The swap timestamps of a board that asks for them: the constant 120 in + * slots 0, 1 and 6, a time that is never in the future. Counter values + * (1 GHz on T6031) make the firmware time out on the first swap. Call after + * dcp_v14_encode_swap(), which zeroes the record. + */ +#define DCP_V14_SWAP_TIMESTAMP_VALUE 120 + +static inline void dcp_v14_swap_timestamps(u8 *swap) +{ + put_unaligned_le64(DCP_V14_SWAP_TIMESTAMP_VALUE, swap + DCP_V14_SWAP_TIMESTAMP(0)); + put_unaligned_le64(DCP_V14_SWAP_TIMESTAMP_VALUE, swap + DCP_V14_SWAP_TIMESTAMP(1)); + put_unaligned_le64(DCP_V14_SWAP_TIMESTAMP_VALUE, swap + DCP_V14_SWAP_TIMESTAMP(6)); +} + #endif /* __APPLE_IOMFB_V14_7_SWAP_H__ */ diff --git a/drivers/gpu/drm/apple/v14-board-test.c b/drivers/gpu/drm/apple/v14-board-test.c index 801ed2f623a4c5..98df7dfab36ea8 100644 --- a/drivers/gpu/drm/apple/v14-board-test.c +++ b/drivers/gpu/drm/apple/v14-board-test.c @@ -3,12 +3,16 @@ * The 14.x IOMFB board records: which record a DCP gets on which machine, * and which DCP firmware image a record admits. The J516C (T6031) record * admits only the T6030 image, whose IOMFB layouts were validated, and - * refuses every other; the existing records are unchanged. + * refuses every other; the existing records are unchanged. Which DCPs stamp + * their swaps: the internal panel of a T6031 record; the T6030 and T8122 + * panels and external processors keep zero timestamps, byte for byte. */ #include #include +#include #include "iomfb_v14_7_board.h" +#include "iomfb_v14_7_swap.h" #define T6030_DCP_IMAGE "DDF38191-93B3-324A-BC8F-643006F5AC82" #define J613_DCP_IMAGE "90F849E1-B422-367E-B389-50246F8DEC47" @@ -71,6 +75,7 @@ static void v14_board_j516c(struct kunit *test) /* No colour matrix method is known for this board. */ KUNIT_EXPECT_EQ(test, b->ctm_set, 0); KUNIT_EXPECT_EQ(test, b->ctm_get, 0); + KUNIT_EXPECT_TRUE(test, b->base_edid); } /* @@ -95,6 +100,8 @@ static void v14_board_j514c(struct kunit *test) KUNIT_EXPECT_TRUE(test, b->promotion); KUNIT_EXPECT_TRUE(test, b->log_bringup); KUNIT_EXPECT_EQ(test, b->ctm_set, 0); + /* Its panel is not the J516C's: no base EDID. */ + KUNIT_EXPECT_FALSE(test, b->base_edid); KUNIT_EXPECT_TRUE(test, dcp_v14_board_admits(b, J516C_DCP_IMAGE)); KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); /* Not the J516C record. */ @@ -153,6 +160,7 @@ static void v14_board_unchanged(struct kunit *test) KUNIT_EXPECT_TRUE(test, b->promotion); KUNIT_EXPECT_FALSE(test, b->log_bringup); KUNIT_EXPECT_EQ(test, b->ctm_set, 0); + KUNIT_EXPECT_FALSE(test, b->base_edid); KUNIT_EXPECT_TRUE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, J613_DCP_IMAGE)); /* The T6030 record serves a T6030 DCP on any machine, as before. */ @@ -167,6 +175,7 @@ static void v14_board_unchanged(struct kunit *test) KUNIT_EXPECT_EQ(test, b->panel_height, 1664); KUNIT_EXPECT_FALSE(test, b->promotion); KUNIT_EXPECT_FALSE(test, b->log_bringup); + KUNIT_EXPECT_FALSE(test, b->base_edid); KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); /* A T8122 DCP on a T6031 machine has no record. */ @@ -174,12 +183,83 @@ static void v14_board_unchanged(struct kunit *test) KUNIT_EXPECT_TRUE(test, known); } +/* Only the internal panel of the J516C record stamps its swaps. */ +static void v14_board_swap_timestamps(struct kunit *test) +{ + const struct dcp_v14_board *j516c_b, *j514c_b, *t6030_b, *j613_b; + bool known; + + j516c_b = board_for("apple,t6031-dcp", j516c, &known); + j514c_b = board_for("apple,t6031-dcp", j514c, &known); + t6030_b = board_for("apple,t6030-dcp", j516s, &known); + j613_b = board_for("apple,t8122-dcp", j613, &known); + KUNIT_ASSERT_TRUE(test, j516c_b && j514c_b && t6030_b && j613_b); + + KUNIT_EXPECT_TRUE(test, j516c_b->swap_timestamps); + KUNIT_EXPECT_FALSE(test, j514c_b->swap_timestamps); + KUNIT_EXPECT_FALSE(test, t6030_b->swap_timestamps); + KUNIT_EXPECT_FALSE(test, j613_b->swap_timestamps); + + KUNIT_EXPECT_TRUE(test, dcp_v14_board_stamps_swaps(j516c_b, false)); + /* The J514C, the external role, the T6030 (J514S, J516S) and T8122 panels, no record. */ + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(j514c_b, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(j516c_b, true)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(t6030_b, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(j613_b, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(NULL, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(NULL, true)); +} + +/* + * A stamped swap holds 120 in timestamp slots 0, 1 and 6 and is otherwise the + * encoded record, with or without a surface; an unstamped one keeps every + * slot zero. + */ +static void v14_swap_timestamps_encoding(struct kunit *test) +{ + const size_t end = DCP_V14_SWAP_TIMESTAMP(DCP_V14_SWAP_TIMESTAMPS); + u8 surface[DCP_V14_SURFACE_SIZE]; + unsigned int i, pass; + u8 *plain, *stamped; + + plain = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + stamped = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, plain && stamped); + dcp_v14_encode_surface(surface, 3456 * 4, 3456, 2160, true); + + for (pass = 0; pass < 2; pass++) { + const u8 *s = pass ? surface : NULL; + + dcp_v14_encode_swap(plain, 0x2a, 0xff000000, s, 0x10000, pass ? 3456 : 0, + pass ? 2160 : 0, 74, 0); + dcp_v14_encode_swap(stamped, 0x2a, 0xff000000, s, 0x10000, pass ? 3456 : 0, + pass ? 2160 : 0, 74, 0); + dcp_v14_swap_timestamps(stamped); + + for (i = 0; i < DCP_V14_SWAP_TIMESTAMPS; i++) { + size_t at = DCP_V14_SWAP_TIMESTAMP(i); + u64 want = i == 0 || i == 1 || i == 6 ? 120 : 0; + + KUNIT_EXPECT_EQ_MSG(test, get_unaligned_le64(plain + at), 0, + "surface %u: unstamped slot %u", pass, i); + KUNIT_EXPECT_EQ_MSG(test, get_unaligned_le64(stamped + at), want, + "surface %u: slot %u", pass, i); + } + /* The rest of the record, from the word after slot 6, is unchanged. */ + KUNIT_EXPECT_EQ(test, + memcmp(plain + end, stamped + end, DCP_V14_SWAP_SIZE - end), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(stamped + DCP_V14_SWAP_ID), 0x2a); + } +} + static struct kunit_case v14_board_cases[] = { KUNIT_CASE(v14_board_j516c), KUNIT_CASE(v14_board_j514c), KUNIT_CASE(v14_board_t6031_other_boards), KUNIT_CASE(v14_board_j516c_image), KUNIT_CASE(v14_board_unchanged), + KUNIT_CASE(v14_board_swap_timestamps), + KUNIT_CASE(v14_swap_timestamps_encoding), {} }; diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 958600a91b06bd..9183073cd92236 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -42,6 +42,7 @@ config DRM_ASAHI_G15 depends on DRM_APPLE=y || (DRM_APPLE=m && DRM_ASAHI=m) select APPLE_T6030_GPU_PMP_LINK select CRYPTO_LIB_SHA256 + select RUST_FW_LOADER_ABSTRACTIONS default n help Enable the native G15 backend for explicitly admitted M3 Pro/Max diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 73a798fc69c00c..a55322d50ad236 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -137,7 +137,23 @@ module! { }, g15_pb_max_blocks: u32 { default: 3072, - description: "G15: parameter-buffer growth maximum given to the firmware, blocks of 128 KiB (1024..4095 in effect; the blocks above it, up to 4095, answer partial renders; the 512 MiB heap is allocated either way)", + description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", + }, + g15_pb_blocks: u32 { + default: 64, + description: "G15: PB blocks posted at (re)initialization, 128 KiB each (64..4095)", + }, + g15_pb_reclaim: u32 { + default: 2, + description: "G15: PB reclaim when idle: 0 never, 1 re-initialize, 2 also free backing", + }, + g15_pb_idle_ms: u32 { + default: 1000, + description: "G15: idle time before a PB reclaim, ms (at least 100)", + }, + g15_pb_reclaim_blocks: u32 { + default: 1024, + description: "G15: PB blocks posted above which an idle engine reclaims the PB", }, g15_early_tiling: u32 { default: 0, diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index e6278690d5f14a..cef9cc8b330059 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -63,6 +63,7 @@ pub(crate) mod glue { fn asahi_g15_release_power(dev: *mut core::ffi::c_void); fn asahi_g15_module_get(); pub(crate) fn asahi_g15_reprobe(); + pub(crate) fn asahi_g15_report_loader_gpu(); fn asahi_g15_adt_phram(start: *mut u64, size: *mut u64) -> core::ffi::c_int; fn asahi_g15_ram_disjoint(start: u64, size: u64) -> bool; fn asahi_g15_ram_within(start: u64, size: u64) -> bool; diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index e5c18a15e0c3b3..33974222afe9b0 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -515,8 +515,12 @@ impl Firmware { if self.engine.flights.capacity() < sizes::FLIGHTS { self.engine.flights.reserve(sizes::FLIGHTS, GFP_KERNEL)?; } + // A job arriving at an idle engine: the PB reclaim, if due, happens before its + // publication (idle polls are not re-armed, so this is where a long idle ends). + self.reclaim_check(); self.engine.pending.push(job.clone(), GFP_KERNEL)?; job.progress.set_pending(true); + self.note_idle(); let q = &job.queue; let owned = self.engine.queue_published(&job) + self @@ -601,6 +605,9 @@ impl Firmware { } } } + // A publication that left nothing published and nothing pending (jobs that failed + // before any visible record) starts the idle. + self.note_idle(); result } diff --git a/drivers/gpu/drm/asahi/g15/gate.rs b/drivers/gpu/drm/asahi/g15/gate.rs index f41b2e3f75f57c..a6fb20bf156b5c 100644 --- a/drivers/gpu/drm/asahi/g15/gate.rs +++ b/drivers/gpu/drm/asahi/g15/gate.rs @@ -3,11 +3,13 @@ //! Stream admission gates (safety nets for userspace encoders). //! //! VDM: the root stream's first word is 0x21000960. CDM walk from `cdm_ctrl_stream_base` through -//! links until TERMINATE (0x40000000), at most 8192 records and 64 links (a longer or cyclic -//! stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; +//! links until TERMINATE (0x40000000), at most 64 links and [`CDM_MAX_BYTES`] of records (a +//! longer or cyclic stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; //! link 0x2000XXXX; direct launch (40 B, the G15 layout), indirect launches 36 B //! (`w0 & 0xff7fffff == 0x08080000`) and 24 B (0x10080000); every program VA inside the queue's -//! 4 GiB USC window, 64-aligned, first 64 bytes readable. Client memory is read through the VM's +//! 4 GiB USC window, 64-aligned, first 64 bytes readable; an indirect launch's 12 argument bytes +//! readable. A program or argument page found good is not checked again in the same walk, and a +//! walk checks at most [`CDM_MAX_PAGE_CHECKS`] pages. Client memory is read through the VM's //! page tables with a point-of-coherency clean+invalidate of the lines ([`mmu::Vm::read_user`]: //! the GPU may have written it). `g15_stream_gate=0` disables both gates (encoder bring-up only). @@ -24,8 +26,19 @@ const CDM_LINK: u32 = 0x20000000; const CDM_DIRECT: u32 = 0x00080000; const CDM_INDIRECT_SIZES: u32 = 0x08080000; const CDM_INDIRECT: u32 = 0x10080000; -const CDM_MAX_RECORDS: u32 = 8192; const CDM_MAX_LINKS: u32 = 64; +/// The stream block the encoder links to when one fills (Mesa's 64 KiB control-stream blocks). +const CDM_BLOCK_BYTES: u64 = 0x10000; +/// Stream bytes walked at most: the root block and one block per admitted link. A stream of +/// that size holds every record the encoder can write before the link budget (a record count +/// does not: the 1024 tessellated draws of one multi-draw write more than 8192). +const CDM_MAX_BYTES: u64 = (CDM_MAX_LINKS as u64 + 1) * CDM_BLOCK_BYTES; +/// Page checks (a program's or an indirect launch's arguments, each under the VM lock) a walk +/// makes at most. The last page found good is not checked again, but the encoder's indirect +/// arguments move to a new page every few launches: a stream at the link budget makes about +/// 8100. Twice that leaves room; without a bound, a stream of distinct pages would make one per +/// 24-byte launch, about 177000 in the byte budget. +const CDM_MAX_PAGE_CHECKS: u32 = 16384; /// Rejections logged per boot. const MAX_REPORTS: u32 = 32; @@ -86,19 +99,52 @@ impl Window { } } +/// The pages a walk has found good and the page checks it has made. +struct Checked { + program: Option, + args: Option, + count: u32, +} + +/// One page check of `[start, end)` under the walk's page budget: ENOSPC once the budget is +/// spent. +fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result { + if *count >= CDM_MAX_PAGE_CHECKS { + return Err(ENOSPC); + } + *count += 1; + cover(vm, start, end, true, false) +} + /// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable. -/// `checked` caches the last proven program page. -fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Option) -> Result { +/// The last proven program page is not checked again. +fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Result { if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 { return Err(EINVAL); } // 64-aligned: the block never crosses its page. let page = address & !(mmu::UAT_PGMSK as u64); - if *checked == Some(page) { + if checked.program == Some(page) { return Ok(()); } - cover(vm, address, address + 64, true, false)?; - *checked = Some(page); + check(vm, address, address + 64, &mut checked.count)?; + checked.program = Some(page); + Ok(()) +} + +/// An indirect launch's 12 argument bytes: readable. The last proven argument page is not +/// checked again; arguments that cross a page are checked each time. +fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result { + let end = address.checked_add(12).ok_or(EINVAL)?; + let page = address & !(mmu::UAT_PGMSK as u64); + let within = (end - 1) & !(mmu::UAT_PGMSK as u64) == page; + if within && checked.args == Some(page) { + return Ok(()); + } + check(vm, address, end, &mut checked.count)?; + if within { + checked.args = Some(page); + } Ok(()) } @@ -128,15 +174,19 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { len: 0, bytes: [0; 256], }; - let mut checked = None; + let mut checked = Checked { + program: None, + args: None, + count: 0, + }; let mut va = base; let mut links = 0; - let mut records = 0; - while records < CDM_MAX_RECORDS { - records += 1; + let mut walked = 0; + while walked < CDM_MAX_BYTES { let w = window.words(vm, va, 1)?; if CDM_BARRIERS.contains(&w[0]) { va += 4; + walked += 4; continue; } if w[0] == CDM_TERMINATE { @@ -152,6 +202,7 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { if va & 3 != 0 { return Err(reject("CDM link", va, &w)); } + walked += 8; continue; } if w[0] == CDM_DIRECT { @@ -162,9 +213,13 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { } match program(vm, usc, address, &mut checked) { Err(error) if error == EINVAL => return Err(reject("CDM program", va, &w)), + Err(error) if error == ENOSPC => { + return Err(reject("CDM page check budget", va, &w)) + } result => result?, } va += 40; + walked += 40; continue; } let kind = w[0] & 0xff7fffff; @@ -182,15 +237,20 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { return Err(reject("CDM indirect launch", va, &w)); } let result = program(vm, usc, address, &mut checked) - .and_then(|()| cover(vm, indirect, indirect + 12, true, false)); + .and_then(|()| arguments(vm, indirect, &mut checked)); match result { Err(error) if error == EINVAL => return Err(reject("CDM indirect", va, &w)), + Err(error) if error == ENOSPC => { + return Err(reject("CDM page check budget", va, &w)) + } result => result?, } - va += if sizes { 36 } else { 24 }; + let size = if sizes { 36 } else { 24 }; + va += size; + walked += size; continue; } return Err(reject("CDM", va, &w)); } - Err(reject("CDM record budget", va, &[])) + Err(reject("CDM byte budget", va, &[])) } diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index 662c9c8258e213..5ceb20cd35d4b8 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -28,6 +28,7 @@ void asahi_g15_hold_power(struct device *dev); void asahi_g15_release_power(struct device *dev); void asahi_g15_module_get(void); void asahi_g15_reprobe(void); +void asahi_g15_report_loader_gpu(void); int asahi_g15_adt_phram(u64 *start, u64 *size); bool asahi_g15_ram_disjoint(u64 start, u64 size); bool asahi_g15_ram_within(u64 start, u64 size); @@ -39,6 +40,47 @@ int asahi_g15_pmp_get(struct device *dev); void asahi_g15_pmp_put(struct device *dev); bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); +/* + * The boot loader enables the T6030/T6031 GPU node only when its + * chosen.asahi,t603x-gpu setting is on; otherwise the node stays disabled, + * the adapter never probes and nothing says why native G15 is absent. Name + * the setting once, at module activation, on a machine whose /chosen has it + * and whose GPU node is disabled. Nothing else changes. + */ +void asahi_g15_report_loader_gpu(void) +{ + static const struct { + const char *prop; + const char *compatible; + } socs[] = { + { "asahi,t6031-gpu", "apple,agx-t6031" }, + { "asahi,t6030-gpu", "apple,agx-t6030" }, + }; + struct device_node *chosen, *np; + const char *value; + unsigned int i; + u32 cell; + + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return; + for (i = 0; i < ARRAY_SIZE(socs); i++) { + if (!of_property_present(chosen, socs[i].prop)) + continue; + np = of_find_compatible_node(NULL, NULL, socs[i].compatible); + if (np && !of_device_is_available(np)) { + if (!of_property_read_string(chosen, socs[i].prop, &value)) + pr_info("asahi: G15: native G15 not started: the boot loader left the GPU node %pOF disabled (/chosen/%s = \"%s\")\n", + np, socs[i].prop, value); + else if (!of_property_read_u32(chosen, socs[i].prop, &cell)) + pr_info("asahi: G15: native G15 not started: the boot loader left the GPU node %pOF disabled (/chosen/%s = %u)\n", + np, socs[i].prop, cell); + } + of_node_put(np); + } + of_node_put(chosen); +} + /* * The maintained m1n1 handoff keeps these four UAT regions in the linear map. * Require the exact enabled reserved-memory owner, in addition to the ADT diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index d566a54953e072..c7f21a9110cee8 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -571,11 +571,10 @@ pub(crate) mod tuning { pub(crate) const SLOT_ROTATE: bool = true; /// Flights per VM. pub(crate) const MAX_INFLIGHT: u32 = 2; - /// PB growth through event 7 / DEVCTL 7, initial and maximum blocks. + /// PB growth through event 7 / DEVCTL 7; the initial posting is `g15_pb_blocks` (64). pub(crate) const PB_GROW: bool = true; - /// Initial PB blocks, and the most that may be posted: one below the mapped heap's block - /// count, so the block-list producer (modulo K = 4096) never returns to 0. - pub(crate) const PB_BLOCKS: u32 = 1024; + /// The most PB blocks that may be posted: one below the mapped heap's block count, so the + /// block-list producer (modulo K = 4096) never returns to 0. pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS - 1; /// At the host deadline, arm the firmware kill. pub(crate) const HANG_PREEMPT: bool = true; diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 75d3ecc30ffe5a..faec77452ffc41 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -222,6 +222,8 @@ static MODULE_READY: AtomicBool = AtomicBool::new(false); /// No fallible module initialization may follow this call. pub(crate) fn activate() { MODULE_READY.store(true, Ordering::Release); + // SAFETY: The helper only reads the device tree. + unsafe { boot::glue::asahi_g15_report_loader_gpu() }; // SAFETY: Both pinned platform registrations are fully initialized and remain owned // by DriverModule. The C helper borrows their live driver pointer only for this call. unsafe { boot::glue::asahi_g15_reprobe() }; diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index 6eabb033a0edf1..48d21e45d0acd6 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -64,13 +64,34 @@ pub(crate) fn accept_text_prefix() -> u64 { } /// The parameter for the PB growth maximum the firmware is given, in blocks, clamped to -/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial 1024 blocks; -/// the blocks above it, up to the 4095 posted at most, answer the synchronous requests of -/// partial renders. It is read-only after boot and does not change the 512 MiB heap. +/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial posting; the +/// blocks above it, up to the 4095 posted at most, answer the synchronous requests of partial +/// renders. It is read-only after boot and does not change the heap's backing. pub(crate) fn pb_max_blocks() -> u32 { (*module_parameters::g15_pb_max_blocks.value()).clamp(64, super::hw::tuning::PB_MAX_BLOCKS) } +/// The initial PB posting, in blocks (64..=4095; the ledger clamps it again). Read-only after +/// boot. +pub(crate) fn pb_blocks() -> u32 { + (*module_parameters::g15_pb_blocks.value()).clamp(64, super::hw::tuning::PB_MAX_BLOCKS) +} + +/// PB reclaim at idle: 0 never, 1 re-initialize, 2 re-initialize and free the backing. +pub(crate) fn pb_reclaim() -> u32 { + (*module_parameters::g15_pb_reclaim.value()).min(2) +} + +/// Idle time before a PB reclaim, ns (at least 100 ms). +pub(crate) fn pb_idle_ns() -> u64 { + (*module_parameters::g15_pb_idle_ms.value()).max(100) as u64 * 1_000_000 +} + +/// Posted blocks above which an idle engine reclaims the PB. +pub(crate) fn pb_reclaim_blocks() -> u32 { + *module_parameters::g15_pb_reclaim_blocks.value() +} + /// Whether FRAGMENT_BARRIERS is advertised and mapped to early tiling. pub(crate) fn early_tiling() -> bool { *module_parameters::g15_early_tiling.value() != 0 diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 751f19865436f8..751bccd07b119e 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -23,7 +23,7 @@ use kernel::{ hrtimer::{ HrTimerCallback, HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode, }, - Delta, Jiffies, Monotonic, + msecs_to_jiffies, Delta, Jiffies, Monotonic, }, workqueue::RawWorkItem, }; @@ -85,7 +85,15 @@ impl Shared { let evented = self.evented.swap(false, Ordering::AcqRel); match firmware.poll(evented, &mut done) { Ok(true) => self.rearm(firmware.engine.chan_wait), - Ok(false) => {} + // Idle: nothing polls until new work arrives, so a PB reclaim that comes + // due with more idle time gets one more pass then. A job's arrival brings + // the poll work forward (`rearm`), which replaces this expiry. + Ok(false) => { + if let Some(ns) = firmware.reclaim_delay_ns() { + let ms = ns.div_ceil(1_000_000).min(u32::MAX as u64) as u32; + self.mod_poll(msecs_to_jiffies(ms) + 1); + } + } Err(error) => self.fail_stop(error), } } @@ -207,6 +215,9 @@ impl Firmware { // prefixes published the retirement loop below drains first. if self.engine.flights.is_empty() { self.drain_rings()?; + // Idle bookkeeping of the PB: when the idle began, and the reclaim once it is due. + self.note_idle(); + self.reclaim_check(); if self.recovery_idle() { return Ok(false); } @@ -228,6 +239,9 @@ impl Firmware { self.engine.prof.polls += 1; self.engine.prof.poll_ns += now_ns().wrapping_sub(start); } + // The pass that retired the last flight is not re-armed: note the idle here, not only + // at the start of a pass that finds the flights empty. + self.note_idle(); published?; Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen) } diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index c635ed11c4a0c3..3364b136b9287f 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -6,16 +6,30 @@ //! (host-tested). //! //! - One 512 MiB PB heap at [`super::hw::client::PB_HEAP`] (4096 blocks of 128 KiB), backed on -//! demand: the initial blocks when the GPU starts, then chunks of at most -//! [`growth::CHUNK_BLOCKS`] (4 MiB) as growth needs them ([`Firmware::back_pb`]), never freed -//! or shrunk. The backing ([`PbChunk`]) is mapped only into user VMs, at the fixed client GVA, -//! each VM's before its prefixes are published ([`Firmware::map_pb`]); nothing names its pages -//! through the host window or context 0 (the records hold client GVAs only). Initial -//! [`super::hw::tuning::PB_BLOCKS`] (1024), at most [`super::hw::tuning::PB_MAX_BLOCKS`] -//! (4095) posted, so the producer never returns to 0; the firmware is given -//! [`Pools::soft_max`] (default 3072) as its growth maximum. Truthful K = 4096 in every +//! demand in chunks of [`growth::CHUNK_BLOCKS`] blocks (16 MiB) from the heap base: when the +//! GPU starts, the chunks of the initial posting and one reserve chunk +//! ([`growth::Ledger::chunks_for`]). The backing ([`PbChunk`]) is mapped only into user VMs, at +//! the fixed client GVA: each VM's before its prefixes are published ([`Firmware::map_pb`]), +//! and a chunk backed later also into the VM of the flights; nothing names its pages through +//! the host window or context 0 (the records hold client GVAs only). Initial +//! `g15_pb_blocks` (64), at most [`super::hw::tuning::PB_MAX_BLOCKS`] (4095) posted, so the +//! producer never returns to 0; the firmware is given [`Pools::soft_max`] (default 3072) as +//! its growth maximum. Truthful K = 4096 in every //! profile (PB reconstruction, also in recovery, synchronizes 8 * K bytes of the block list: //! K = 0x26cd overran the 0x8000-byte list and crashed the second recovery). +//! - Backing: a request is granted from the backed blocks above the posted count; after the +//! replies of a pass, while fewer than a reserve chunk remain, [`Firmware::refill_backing`] +//! backs the next chunk, so a reply never waits for the allocator. Only a reserve that ran dry +//! makes a request wait: it stays queued, the pass backs a chunk and the next pass answers it. +//! - Reclaim ([`Firmware::reclaim_check`], `g15_pb_reclaim`): posted blocks never come back +//! while a PB lives (the firmware recycles pages inside its own pool; the host has no view of +//! it), so an engine idle for `g15_pb_idle_ms` with more than `g15_pb_reclaim_blocks` posted +//! re-initializes the PB at its next render through the re-initialization path of a recovered +//! abort (fresh lists and manager, InitBM of the initial blocks, no halt) and frees the backing +//! above the kept chunks (also with fewer posted, when more is backed than they need); an +//! engine idle for 50 ms (a client boundary) with more than half the maximum posted +//! re-initializes it the same way but keeps the backing; if no render follows, with +//! `g15_pb_reclaim` 2 the backing above the kept chunks is freed after `g15_pb_idle_ms`. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -41,7 +55,6 @@ mod growth; use core::fmt; -use core::sync::atomic::Ordering; use kernel::{ device, prelude::*, @@ -58,7 +71,7 @@ use super::Firmware; use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{gem, mmu}; -pub(crate) use growth::Ledger; +pub(crate) use growth::{Answer, IdleClock, Ledger}; static_assert!(buffer::MAX_BLOCKS == sizes::PB_BLOCKS); static_assert!(buffer::MAX_PAGES == sizes::PB_PAGES); @@ -68,54 +81,43 @@ static_assert!(buffer::MAX_BLOCKS as u64 * buffer::BLOCK_BYTES == client::PB_HEA static_assert!(buffer::PAGE_LIST_SIZE as u64 == client::PB_PAGE_LIST_SIZE); static_assert!(uma_manager::SEEDS == sizes::FREELIST_SEED); static_assert!(uma_pools::POOL_SIZE == sizes::POOL_SIZE); -// The ledger keeps the driver's PB sizes: neither is capped by [`growth::MAX_POSTED`]. +// The ledger keeps the driver's PB maximum: it is not capped by [`growth::MAX_POSTED`]. The +// initial posting (`g15_pb_blocks`) is clamped to MIN_BLOCKS..=PB_MAX_BLOCKS. static_assert!(tuning::PB_MAX_BLOCKS <= growth::MAX_POSTED); -static_assert!(buffer::MIN_BLOCKS <= tuning::PB_BLOCKS); -static_assert!(tuning::PB_BLOCKS <= tuning::PB_MAX_BLOCKS); +static_assert!(buffer::MIN_BLOCKS <= tuning::PB_MAX_BLOCKS); // The PB alias is GPU-only (not cached and firmware-visible): dropping one is a plain unmap, // with no firmware cache flush (`KernelMapping::drop`). static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent()); +static_assert!(growth::MAX_CHUNKS * growth::CHUNK_BLOCKS == buffer::MAX_BLOCKS); -/// Physical backing of PB blocks `[first, first + blocks)`: a zeroed GEM kernel object, pinned -/// and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at the blocks' fixed -/// client GVA ([`Firmware::map_pb`]). Retained until reboot, like every firmware object; each -/// alias also holds the object. +/// Bytes of one backing chunk. +pub(crate) const CHUNK_BYTES: usize = growth::CHUNK_BLOCKS as usize * buffer::BLOCK_BYTES as usize; + +/// Client GVA of backing chunk `i`. +pub(crate) const fn chunk_gva(i: usize) -> u64 { + client::PB_HEAP + (i * CHUNK_BYTES) as u64 +} + +/// Physical backing of one chunk of PB blocks: a zeroed GEM kernel object of [`CHUNK_BYTES`], +/// pinned and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at its fixed +/// client GVA ([`chunk_gva`], [`VmState::map_heap`]). Each alias also holds the object. struct PbChunk { - first: u32, - blocks: u32, _pages: HeldPages, backing: gem::ObjectRef, } impl PbChunk { - fn new(dev: &AsahiDevice, first: u32, blocks: u32) -> Result { - if blocks == 0 || first.checked_add(blocks).ok_or(EINVAL)? > buffer::MAX_BLOCKS { - return Err(EINVAL); - } - let (backing, pages) = - object::gpu_backing(dev, blocks as usize * buffer::BLOCK_BYTES as usize)?; + fn new(dev: &AsahiDevice) -> Result { + let (backing, pages) = object::gpu_backing(dev, CHUNK_BYTES)?; Ok(Self { - first, - blocks, _pages: pages, backing, }) } - /// The block after the chunk's last. - fn end(&self) -> u32 { - self.first + self.blocks - } - - /// Maps the whole chunk, exactly its object's size, at its fixed client GVA in `vm`. - fn map(&self, vm: &mmu::Vm) -> Result { - vm.map_at( - client::PB_HEAP + self.first as u64 * buffer::BLOCK_BYTES, - self.blocks as usize * buffer::BLOCK_BYTES as usize, - &self.backing.gem, - mmu::Prot::from_pte(pte::LEAF_CACHED), - false, - ) + /// The GEM object, for the VM mappings. + fn gem(&self) -> ARef { + self.backing.gem.clone() } } @@ -161,6 +163,13 @@ pub(crate) struct PbStats { pub(crate) grown: u64, pub(crate) refused: u64, pub(crate) bad: u64, + /// Requests that waited a pass for backing. + pub(crate) waits: u64, + /// Chunks backed after the GPU started. + pub(crate) refills: u64, + /// Idle re-initializations, and the chunks they freed. + pub(crate) reclaims: u64, + pub(crate) freed: u64, } /// Render allocator and PB state: the allocators, the PB ledger and the spare set. @@ -179,9 +188,8 @@ pub(crate) struct Pools { spare_ready: bool, pb_pages: Arc, pb_blocks: Arc, - /// The PB backing, contiguous from block 0, in block order: the initial blocks, then the - /// growth chunks. Its capacity, reserved up front, is the most the ledger can need - /// ([`growth::max_chunks`]). + /// The PB backing chunks, from the heap base (chunk `i` at [`chunk_gva`]`(i)`). Its + /// capacity, reserved up front, is the whole heap's [`growth::MAX_CHUNKS`]. chunks: KVec, /// The PB growth ledger since the last (re)initialization. pub(crate) ledger: Ledger, @@ -189,6 +197,11 @@ pub(crate) struct Pools { c1_written: bool, /// The unanswered-at-the-maximum warning was given since the last initialization. exhausted_logged: bool, + /// The engine's idle clock (nothing published, nothing pending), for the reclaim. + idle: IdleClock, + /// The pending re-initialization was set by a reclaim ([`Firmware::reclaim_check`]), not by + /// a recovery; cleared by [`Self::init`]. + reclaim_pending: bool, pub(crate) stats: PbStats, } @@ -376,11 +389,24 @@ impl Pools { 0, true, )?; - // Backing for the initial blocks; growth adds the rest. - let ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); - let mut chunks = - KVec::with_capacity(growth::max_chunks(ledger.initial, ledger.max), GFP_KERNEL)?; - chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?; + // Backing for the initial blocks and the reserve; growth adds the rest. + let mut ledger = Ledger::new( + tuning::PB_GROW, + super::params::pb_blocks(), + tuning::PB_MAX_BLOCKS, + ); + let mut chunks = KVec::with_capacity(growth::MAX_CHUNKS as usize, GFP_KERNEL)?; + for _ in 0..Ledger::chunks_for(ledger.initial) { + chunks.push(PbChunk::new(dev)?, GFP_KERNEL)?; + ledger.back_chunk(); + } + dev_info!( + dev.as_ref(), + "G15: PB heap: {} blocks posted at start, {} chunks of {} MiB backed\n", + ledger.initial, + chunks.len(), + CHUNK_BYTES >> 20 + ); Ok(Self { dev: dev.as_ref().as_ref().into(), drm: dev.into(), @@ -394,6 +420,8 @@ impl Pools { ledger, c1_written: false, exhausted_logged: false, + idle: IdleClock::default(), + reclaim_pending: false, stats: PbStats::default(), }) } @@ -451,9 +479,11 @@ impl Pools { // PB: 16384 pages of 0x8000 in 4096 blocks of 0x20000; with growth the first initial // blocks of them, at most 4095 posted, and page limits at the firmware's growth maximum. - // In every profile the block list declares its real 4096-record capacity. - self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + // The backing is kept. In every profile the block list declares its real 4096-record + // capacity. + self.ledger.reinit(); self.exhausted_logged = false; + self.reclaim_pending = false; if tuning::PB_GROW { // C1 names the initial pages again. self.c1_written = false; @@ -470,7 +500,7 @@ impl Pools { } let heap = client::PB_HEAP; let initial = self.ledger.initial; - if initial > self.backed() { + if initial > self.ledger.backed { return Err(EINVAL); } write_records( @@ -524,133 +554,248 @@ impl Pools { super::params::pb_max_blocks().clamp(self.ledger.initial, self.ledger.max) } - /// The blocks with backing, from block 0. - pub(crate) fn backed(&self) -> u32 { - self.chunks.last().map_or(0, PbChunk::end) + /// Maps the backed chunks `vm` does not hold yet ([`VmState::map_heap`]). + fn map_heap(&self, vm: &VmState) -> Result { + vm.map_heap(self.chunks.iter().map(PbChunk::gem)) + } + + /// Frees the backing chunks from index `keep` on (their VM mappings are dropped by the + /// caller); the RAM returns when the last mapping of a chunk is gone. + fn free_chunks(&mut self, keep: usize) -> usize { + let freed = self.chunks.len().saturating_sub(keep); + self.chunks.truncate(keep); + self.ledger.unback_to(self.chunks.len() as u32); + freed + } + + /// Backs one more chunk; the caller maps it where needed before the ledger posts from it. + fn back_chunk(&mut self) -> Result> { + if self.chunks.len() >= growth::MAX_CHUNKS as usize { + return Err(ENOSPC); + } + let chunk = PbChunk::new(&self.drm)?; + let gem = chunk.gem(); + self.chunks.push(chunk, GFP_KERNEL)?; + Ok(gem) } /// The debugfs line `pb: blocks initial/posted/firmware-max/max backed blocks/chunks - /// requests grown refused bad`. + /// requests grown refused bad waits refills reclaims freed` (new keys are appended). pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { writeln!( f, - "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {}", + "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {} waits {} refills {} reclaims {} freed {}", self.ledger.initial, self.ledger.posted, self.soft_max(), self.ledger.max, - self.backed(), + self.ledger.backed, self.chunks.len(), self.stats.requests, self.stats.grown, self.stats.refused, - self.stats.bad + self.stats.bad, + self.stats.waits, + self.stats.refills, + self.stats.reclaims, + self.stats.freed ) } } impl Firmware { - /// Maps the PB backing into `vm` up to its last chunk, before anything of a prefix of the - /// VM is published (the start of `publish_prefix_into`). Render work runs in one VM at a - /// time (`Engine::publish`), so the VM about to run is the only one through which the GPU - /// can reach a posted block next; a VM created or idle while the backing grew catches up - /// here, also a closed VM whose accepted jobs still run. + /// Maps the PB backing into `vm`, before anything of a prefix of the VM is published (the + /// start of `publish_prefix_into`). Render work runs in one VM at a time + /// (`Engine::publish`), so the VM about to run is the only one through which the GPU can + /// reach a posted block next; a VM created or idle while the backing grew catches up here, + /// also a closed VM whose accepted jobs still run. A chunk is mapped only once + /// ([`VmState::map_heap`]); one that fails leaves the chunks mapped before it, and the + /// prefix is not published. The leaf is GPU-only, so dropping an alias is a plain unmap. /// - /// Each chunk's alias joins the VM's driver mappings, held until the VM is torn down, and - /// the VM's count moves past the chunk only then. When that fails, the alias just made is - /// dropped again: nothing of the VM runs before its count covers the chunk, and the leaf is - /// GPU-only, so the drop is a plain unmap. The chunks mapped before stay counted. - /// - /// The one-VM rule is checked first: a flight of another VM, or a count past the chunks, - /// fails with EPROTO before anything is mapped. + /// The one-VM rule is checked first: a flight of another VM fails with EPROTO before + /// anything is mapped. pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result { - let mut mapped = vm.pb_chunks.load(Ordering::Relaxed); - if mapped > self.pools.chunks.len() - || self - .engine - .flights - .iter() - .any(|f| !core::ptr::eq(&*f.job.vm, vm)) - { + let others = self + .engine + .flights + .iter() + .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) + .count(); + if others != 0 { dev_err!( self.dev(), - "PB backing: ctx {} maps {} of {} chunks with {} flights of another VM\n", + "PB backing: ctx {} maps {} chunks with {} flights of another VM\n", vm.slot, - mapped, self.pools.chunks.len(), - self.engine - .flights - .iter() - .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) - .count() + others ); return Err(super::boot::eproto()); } - while let Some(chunk) = self.pools.chunks.get(mapped) { - let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; - aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; - vm.vm.append_driver_mappings(aliases)?; - mapped += 1; - vm.pb_chunks.store(mapped, Ordering::Relaxed); - } - Ok(()) + self.pools.map_heap(vm) } - /// The VM whose render work runs: every flight's (`Engine::publish` puts a flight behind - /// another only for the same VM), and it must be the asking context's. EPROTO otherwise. - fn running_vm(&self, ctx: u32) -> Result> { - let flights = &self.engine.flights; - let vm = flights - .first() - .ok_or_else(super::boot::eproto)? - .job - .vm - .clone(); - if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) { - dev_err!( - self.dev(), - "PB growth: ctx {} is not the one VM in flight (ctx {}, {} flights)\n", - ctx, - vm.slot, - flights.len() - ); - return Err(super::boot::eproto()); + /// Backs chunks until the reserve is whole, mapping each into the VMs of the flights (by + /// the one-VM rule, the VM that runs) before the ledger may post from it; any other VM + /// maps it at its next publication ([`Self::map_pb`]). A failure is logged and leaves the + /// reserve short: requests then wait, and the next pass tries again. + pub(crate) fn refill_backing(&mut self) { + // A pending re-initialization is about to drop the posted count: nothing to back for, + // unless a render published before it still runs (the flights behind a fragment kill + // keep asking for growth until they retire). + let render = (0..self.engine.flights.len()).any(|i| self.receipts(i).renders != 0); + if self.recovery.reinit && !render { + return; + } + while self.pools.ledger.refill_due() { + let result = (|| -> Result { + let gem = self.pools.back_chunk()?; + let i = self.pools.chunks.len() - 1; + for flight in self.engine.flights.iter() { + flight.job.vm.map_chunk(i, gem.clone())?; + } + Ok(()) + })(); + match result { + Ok(()) => { + self.pools.ledger.back_chunk(); + self.pools.stats.refills += 1; + dev_dbg!( + self.dev(), + "PB heap: chunk {} backed ({} of {} blocks posted)\n", + self.pools.chunks.len() - 1, + self.pools.ledger.posted, + self.pools.ledger.backed + ); + } + Err(e) => { + // A chunk that could not be mapped into a flight's VM is not counted: drop + // it (and its aliases in the VMs) again. + let backed = (self.pools.ledger.backed / growth::CHUNK_BLOCKS) as usize; + if self.pools.chunks.len() > backed { + for flight in self.engine.flights.iter() { + flight.job.vm.unmap_heap_above(backed); + } + self.pools.chunks.truncate(backed); + } + dev_warn!( + self.dev(), + "PB heap: backing chunk {} failed: {} ({} of {} blocks posted)\n", + backed, + e.to_errno(), + self.pools.ledger.posted, + self.pools.ledger.backed + ); + return; + } + } } - Ok(vm) } - /// Backs the PB up to block `end` in the VM that runs, the asking context `ctx`, before an - /// answer posts those blocks (nothing new when they are backed already): chunks of at most - /// [`growth::CHUNK_BLOCKS`] blocks are allocated ([`object::gpu_backing`]), mapped into that - /// VM and kept, one at a time. The VM first catches up ([`Self::map_pb`], which also checks - /// the one-VM rule), so its count equals the chunks. A chunk is kept - /// only once its alias in the VM is held, and the VM's count moves with it; when the alias - /// cannot be held, the alias and the chunk are dropped, neither ever reachable (the leaf is - /// GPU-only: a plain unmap). An error leaves the chunks backed so far for the next request, - /// and nothing is posted. The allocation sleeps (shmem pages with `__GFP_RETRY_MAYFAIL`) - /// while the firmware waits for the answer; how long is not bounded here. - fn back_pb(&mut self, end: u32, ctx: u32) -> Result { - let vm = self.running_vm(ctx)?; - self.map_pb(&vm)?; - while self.pools.backed() < end { - let (first, blocks) = - growth::next_chunk(self.pools.backed(), self.pools.ledger.max).ok_or(ENOSPC)?; - if self.pools.chunks.len() == self.pools.chunks.capacity() { - return Err(ENOSPC); + /// Notes the engine idle (nothing published, nothing pending) or busy; the idle time is + /// what [`Self::reclaim_check`] measures. Called at the end of every poll pass and + /// publication (the pass that retires the last flight is not re-armed) and at a job's + /// arrival. + pub(crate) fn note_idle(&mut self) { + let idle = self.engine.flights.is_empty() && self.engine.pending.is_empty(); + self.pools.idle.note(idle, super::engine::now_ns()); + } + + /// Reclaims the PB when the engine has been idle for `g15_pb_idle_ms` with more than + /// `g15_pb_reclaim_blocks` posted or more backing than the posted blocks need, or for + /// [`growth::DRAIN_IDLE_NS`] with more than half the maximum posted + /// ([`growth::Ledger::drain_reinit_due`]; re-initialization only): the next render prefix + /// re-initializes the render allocators and the PB (InitBM of the initial blocks; + /// `recovery.reinit`, as after a recovered abort but with no halt), and with + /// `g15_pb_reclaim` 2 the backing chunks above those of the initial posting and the reserve + /// are freed now, unmapped from every VM of the registry (a VM that publishes later drops + /// its stale mappings at [`VmState::map_heap`]). While a reclaim's re-initialization is + /// pending (a client-boundary one kept the backing), `g15_pb_reclaim` 2 frees the backing + /// above those chunks once the engine has been idle for `g15_pb_idle_ms` + /// ([`growth::Ledger::pending_free_due`]). Called under the device lock when a job arrives + /// at an idle engine and in idle poll passes. + pub(crate) fn reclaim_check(&mut self) { + let level = super::params::pb_reclaim(); + if level == 0 + || !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || (self.recovery.reinit && !self.pools.reclaim_pending) + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + let floor = super::params::pb_reclaim_blocks(); + let min_idle = super::params::pb_idle_ns(); + let ledger = &self.pools.ledger; + if self.recovery.reinit { + if level >= 2 && ledger.pending_free_due(idle_ns, min_idle) { + let posted = ledger.posted; + let freed = self.free_backing(); + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted and the re-initialization pending: {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + freed, + self.pools.chunks.len() + ); } - let chunk = PbChunk::new(&self.pools.drm, first, blocks)?; - let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; - aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; - vm.vm.append_driver_mappings(aliases)?; - // Within the capacity checked above. - self.pools - .chunks - .push_within_capacity(chunk) - .map_err(|_| EIO)?; - vm.pb_chunks - .store(self.pools.chunks.len(), Ordering::Relaxed); + return; } - Ok(()) + let idle = ledger.reclaim_due(floor, idle_ns, min_idle) + || (level >= 2 && ledger.free_due(idle_ns, min_idle)); + if !idle && !ledger.drain_reinit_due(idle_ns) { + return; + } + self.recovery.reinit = true; + self.pools.reclaim_pending = true; + self.pools.stats.reclaims += 1; + let posted = self.pools.ledger.posted; + let mut freed = 0; + // A short idle keeps the backing: the next scene would allocate it again. + if level >= 2 && idle { + freed = self.free_backing(); + } + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted: re-initialization to {} at the next render, {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + self.pools.ledger.initial, + freed, + self.pools.chunks.len() + ); + } + + /// Frees the backing chunks above those a re-initialized heap keeps + /// ([`growth::Ledger::keep_chunks`]) and drops their mappings in every VM that could + /// publish; returns the chunks freed. If the VMs cannot be walked, their mappings keep the + /// pages until each VM drops its stale aliases ([`VmState::map_heap`]): nothing is counted + /// freed. + fn free_backing(&mut self) -> usize { + let keep = self.pools.ledger.keep_chunks() as usize; + let dropped = self.pools.free_chunks(keep); + // The mappings hold the chunks' pages: drop them in every VM that could publish. + let freed = match self.vms.open() { + Ok(states) => { + for state in states.iter() { + state.unmap_heap_above(keep); + } + dropped + } + Err(e) => { + dev_warn!( + self.dev(), + "PB heap: reclaim could not walk the VMs: {}\n", + e.to_errno() + ); + 0 + } + }; + self.pools.stats.freed += freed as u64; + freed } /// (Re)initializes the render UMA managers, PB lists and PB manager. `rebuild`: the priming @@ -683,9 +828,12 @@ impl Firmware { /// DEVCTL 7 reports success; at the ledger's maximum the reply is the refusal (+4 = 0), /// which gives the firmware no new pages, and after two refusals requests are left /// unanswered. The firmware imports the blocks itself: P is never rewritten. Only the - /// requests queued before the pass are taken. A failed reply is logged; a request that - /// cannot be processed at all (a firmware object access failed) is logged and left - /// unanswered like an invalid one. Never fails: the poll pass continues. + /// requests queued before the pass are taken. A request with no backed block left waits: + /// it goes back to the head of the queue for the next pass, and this pass ends its answers + /// there. After the answers the reserve is refilled ([`Self::refill_backing`]). A failed + /// reply is logged; a request that cannot be processed at all (a firmware object access + /// failed) is logged and left unanswered like an invalid one. Never fails: the poll pass + /// continues. pub(crate) fn answer_growth(&mut self) -> Result { // Only the requests queued before this pass: a reply's wait drains the rings and may // queue the next request, which the next pass answers (the device lock is released in @@ -695,23 +843,67 @@ impl Firmware { let Ok(request) = self.events.grow.remove(0) else { break; }; - self.pools.stats.requests += 1; - if let Err(e) = self.answer_request(&request) { - self.pools.stats.bad += 1; - dev_err!( - self.dev(), - "PB growth request {} ctx {} not answered: {}\n", - request[buffer::grow_request::COUNTER], - request[buffer::grow_request::CONTEXT], - e.to_errno() - ); + match self.answer_request(&request) { + Ok(true) => self.pools.stats.requests += 1, + Ok(false) => { + // Waits for backing: back at the head, counted once answered. + self.pools.stats.waits += 1; + // The queue has room: this request was just taken from it. + if self.events.grow.insert_within_capacity(0, request).is_err() { + self.pools.stats.bad += 1; + } + break; + } + Err(e) => { + self.pools.stats.requests += 1; + self.pools.stats.bad += 1; + dev_err!( + self.dev(), + "PB growth request {} ctx {} not answered: {}\n", + request[buffer::grow_request::COUNTER], + request[buffer::grow_request::CONTEXT], + e.to_errno() + ); + } } } + self.refill_backing(); Ok(()) } - /// Processes one growth request. - fn answer_request(&mut self, request: &[u32; 14]) -> Result { + /// How long the idle engine must stay idle before [`Self::reclaim_check`] acts, if it can + /// act without new work ([`growth::Ledger::reclaim_delay`], or with a reclaim's + /// re-initialization pending [`growth::Ledger::pending_free_delay`]); `None` while busy, + /// while a recovery or its re-initialization is pending, or when nothing would become due. + /// An idle engine runs no poll pass by itself, so the poll work is armed for this delay. + pub(crate) fn reclaim_delay_ns(&self) -> Option { + if !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || (self.recovery.reinit && !self.pools.reclaim_pending) + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return None; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + if self.recovery.reinit { + return self.pools.ledger.pending_free_delay( + super::params::pb_reclaim(), + idle_ns, + super::params::pb_idle_ns(), + ); + } + self.pools.ledger.reclaim_delay( + super::params::pb_reclaim(), + super::params::pb_reclaim_blocks(), + idle_ns, + super::params::pb_idle_ns(), + ) + } + + /// Processes one growth request: `Ok(false)` when it waits for backing. + fn answer_request(&mut self, request: &[u32; 14]) -> Result { let ctl = self.init.control.clone().ok_or(ENODEV)?; let epoch = ctl.r32(control::HALT_COUNT)? as u64 | (ctl.r32(control::HALT_COUNT + 4)? as u64) << 32; @@ -737,7 +929,7 @@ impl Firmware { self.pools.ledger.max ); } - return Ok(()); + return Ok(true); } // A request from before a halt has no authority over the PB now. if !self @@ -758,15 +950,15 @@ impl Firmware { live as u32, tuning::PB_GROW as u32 ); - return Ok(()); + return Ok(true); } - // Every block the answer posts is backed in the VM that runs before anything of the - // answer is written; the one-VM rule is checked for every answer. If that fails, nothing - // is posted or answered (answer_growth logs it; the deadline applies), and the backing - // made so far stays for the next request. - let end = self.pools.ledger.posted + self.pools.ledger.next_count(); - self.back_pb(end, ctx)?; - let (first, n) = self.pools.ledger.accept(request); + // A grant posts only backed blocks, and every backed chunk is mapped in the VM of the + // flights (at its publication, or at the refill that backed it). + let (first, n) = match self.pools.ledger.accept(request) { + Answer::Grant { first, count } => (first, count), + Answer::Refuse => (self.pools.ledger.posted, 0), + Answer::Wait => return Ok(false), + }; let heap = client::PB_HEAP; for b in first..first + n { self.pools.pb_blocks.w64( @@ -831,6 +1023,6 @@ impl Firmware { ret ); } - Ok(()) + Ok(true) } } diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 20b298e641459d..fef40a6d3a73e3 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -12,6 +12,15 @@ //! its recovery path then apply, without a guarantee that they end the stall. A request from //! before a halt (epoch) or not matching a published render prefix is left unanswered. //! +//! The heap's VA range covers [`MAX_BLOCKS`] blocks; its RAM comes in chunks of +//! [`CHUNK_BLOCKS`] blocks from the heap base, which the ledger counts as `backed`. A request is +//! granted from the backed blocks above `posted`. A chunk is due ([`Ledger::refill_due`]) +//! whenever fewer than [`RESERVE_BLOCKS`] backed blocks remain above the posted count, so that +//! it can be backed after a reply; a request that finds no backed block waits +//! ([`Answer::Wait`]) and keeps its counter. A re-initialization ([`Ledger::reinit`]) returns +//! the posted count to the initial size and keeps the backing; [`Ledger::keep_chunks`] gives the +//! chunks that the initial posting and the reserve need. +//! //! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. use crate::g15::fw::buffer::{grow_request, BLOCK_LIST_RECORDS, GROW_STEP, MAX_BLOCKS, MIN_BLOCKS}; @@ -26,6 +35,8 @@ pub(crate) struct Ledger { pub(crate) max: u32, /// Blocks posted to the block list, cumulative (the block_ctl+4 producer is this modulo K). pub(crate) posted: u32, + /// Blocks with RAM behind them, from the heap base. + pub(crate) backed: u32, /// The next expected request counter. pub(crate) next_request: u32, /// A refusal at the maximum was logged since the last initialization. @@ -41,26 +52,30 @@ pub(crate) struct Ledger { /// Refusals answered at the maximum before requests are left to the deadline. pub(crate) const MAX_REFUSALS: u32 = 2; -/// Blocks of each PB backing chunk after the initial one (4 MiB): growth backs at most this -/// much at a time while the firmware waits for its answer. -pub(crate) const CHUNK_BLOCKS: u32 = 32; - -/// The backing chunk that follows `backed` blocks of a ledger with maximum `max`: its first block -/// and size, at most [`CHUNK_BLOCKS`] and never past `max`; none from `max` on. -pub(crate) const fn next_chunk(backed: u32, max: u32) -> Option<(u32, u32)> { - if backed >= max { - None - } else if max - backed < CHUNK_BLOCKS { - Some((backed, max - backed)) - } else { - Some((backed, CHUNK_BLOCKS)) - } -} +/// Blocks of each PB backing chunk (16 MiB). +pub(crate) const CHUNK_BLOCKS: u32 = 128; + +/// Backed blocks kept above the posted count: at 1-2 ms per request of ten blocks, 13 requests +/// of lead for the allocation of the next chunk. +pub(crate) const RESERVE_BLOCKS: u32 = CHUNK_BLOCKS; -/// The chunks that back a ledger with `initial` and `max` blocks at most: the initial one and -/// [`next_chunk`]'s from `initial` to `max`. -pub(crate) const fn max_chunks(initial: u32, max: u32) -> usize { - 1 + max.saturating_sub(initial).div_ceil(CHUNK_BLOCKS) as usize +/// Chunks of the whole heap. +pub(crate) const MAX_CHUNKS: u32 = MAX_BLOCKS / CHUNK_BLOCKS; + +/// The idle time before a re-initialization past half the maximum (a client boundary, not a +/// gap between one client's renders). +pub(crate) const DRAIN_IDLE_NS: u64 = 50_000_000; + +/// The answer to an admitted growth request. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Answer { + /// Post blocks `[first, first + count)` (`count` at most [`GROW_STEP`]) and reply grown. + Grant { first: u32, count: u32 }, + /// Reply refused: the maximum is posted. + Refuse, + /// No backed block above the posted count: leave the request queued until a chunk is + /// backed. + Wait, } /// The most blocks a ledger posts: the mapped heap, and below K, so that the block_ctl+4 @@ -85,17 +100,141 @@ impl Ledger { initial, max, posted: initial, + backed: 0, next_request: 0, refusal_logged: false, refusals: 0, } } + /// A re-initialization: the initial blocks are posted again, the request counter and the + /// refusals start over, the backing stays. + pub(crate) const fn reinit(&mut self) { + self.posted = self.initial; + self.next_request = 0; + self.refusal_logged = false; + self.refusals = 0; + } + /// The live block_ctl+4 producer. pub(crate) const fn producer(&self) -> u32 { self.posted % BLOCK_LIST_RECORDS } + /// Records one more backed chunk. + pub(crate) const fn back_chunk(&mut self) { + self.backed += CHUNK_BLOCKS; + } + + /// Drops the backing to `chunks` chunks. + pub(crate) const fn unback_to(&mut self, chunks: u32) { + self.backed = chunks * CHUNK_BLOCKS; + } + + /// Backed blocks above the posted count. + pub(crate) const fn spare(&self) -> u32 { + self.backed.saturating_sub(self.posted) + } + + /// Whether another chunk is due: fewer than the reserve above the posted count and the heap + /// not yet backed whole. + pub(crate) const fn refill_due(&self) -> bool { + self.backed < MAX_BLOCKS && self.spare() < RESERVE_BLOCKS + } + + /// Chunks that `blocks` posted blocks and the reserve need. + pub(crate) const fn chunks_for(blocks: u32) -> u32 { + let n = (blocks + RESERVE_BLOCKS).div_ceil(CHUNK_BLOCKS); + if n > MAX_CHUNKS { + MAX_CHUNKS + } else { + n + } + } + + /// Chunks a re-initialized heap keeps: those of the initial posting and the reserve. + pub(crate) const fn keep_chunks(&self) -> u32 { + Self::chunks_for(self.initial) + } + + /// Whether the heap is to be re-initialized: idle for `idle_ns` with more than `floor` + /// blocks posted, and grown since the initialization (with nothing grown a + /// re-initialization posts the same blocks again). + pub(crate) const fn reclaim_due(&self, floor: u32, idle_ns: u64, min_idle_ns: u64) -> bool { + self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns + } + + /// Whether the backing is to be freed: idle for `idle_ns` with more chunks backed than the + /// posted blocks and the reserve need. A client-boundary re-initialization keeps the + /// backing of a heavy scene; if the next client posts below the reclaim floor, this frees it. + pub(crate) const fn free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS + } + + /// While a reclaim's re-initialization is pending (the posted count returns to the initial + /// size at the next render): whether the backing above [`Self::keep_chunks`] is to be + /// freed, idle for `min_idle_ns`. A client-boundary re-initialization keeps the backing; + /// with no render after it, this frees it. + pub(crate) const fn pending_free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > self.keep_chunks() * CHUNK_BLOCKS + } + + /// How much longer the engine must stay idle, from `idle_ns`, before + /// [`Self::pending_free_due`] with `level`; `None` when it cannot become due (level below + /// 2, or nothing above the kept chunks). 0 when it is due now. + pub(crate) const fn pending_free_delay( + &self, + level: u32, + idle_ns: u64, + min_idle_ns: u64, + ) -> Option { + if level >= 2 && self.backed > self.keep_chunks() * CHUNK_BLOCKS { + Some(min_idle_ns.saturating_sub(idle_ns)) + } else { + None + } + } + + /// Whether a short idle re-initializes the heap: more than half the maximum posted and + /// idle for [`DRAIN_IDLE_NS`]. Back-to-back heavy scenes (a new client every few hundred + /// ms) never reach `g15_pb_idle_ms`, and posted blocks never come back while a PB lives, so + /// without this they run the heap to the maximum. A re-initialization with no idle time + /// faulted at the heap base (at a client switch, the second in a row with no recovery + /// between); with 50 ms of idle none did, also between two renders of one client. + pub(crate) const fn drain_reinit_due(&self, idle_ns: u64) -> bool { + self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS + } + + /// How much longer the engine must stay idle, from `idle_ns`, before a reclaim with + /// `level` is due ([`Self::reclaim_due`] with `floor` and `min_idle_ns`, [`Self::free_due`] + /// at level 2, or [`Self::drain_reinit_due`]); `None` when none can become due while the + /// ledger stays as it is. 0 when one is due now. + pub(crate) const fn reclaim_delay( + &self, + level: u32, + floor: u32, + idle_ns: u64, + min_idle_ns: u64, + ) -> Option { + if level == 0 { + return None; + } + let grown = self.posted > self.initial; + let mut due = u64::MAX; + if grown && self.posted > self.max / 2 { + due = DRAIN_IDLE_NS; + } + let free = level >= 2 && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS; + if ((grown && self.posted > floor) || free) && min_idle_ns < due { + due = min_idle_ns; + } + if due == u64::MAX { + None + } else { + Some(due.saturating_sub(idle_ns)) + } + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -114,21 +253,24 @@ impl Ledger { && request_epoch(request) == epoch } - /// The blocks the next accepted request posts: at most [`GROW_STEP`], none at the maximum. - pub(crate) fn next_count(&self) -> u32 { - GROW_STEP.min(self.max - self.posted) - } - - /// Accepts an admitted request: the next expected counter follows it, and the returned - /// blocks `[first, first + count)` are to be written to the block list, `count` 0 meaning a - /// refusal at the maximum. - pub(crate) fn accept(&mut self, request: &[u32; 14]) -> (u32, u32) { + /// Accepts an admitted request. Granted (at most [`GROW_STEP`] blocks, and no more than are + /// backed) or refused at the maximum, the next expected counter follows it. A request that + /// finds no backed block waits: its counter is not consumed, and it is accepted again once a + /// chunk is backed. + pub(crate) fn accept(&mut self, request: &[u32; 14]) -> Answer { + let room = self.max - self.posted; + if room != 0 && self.spare() == 0 { + return Answer::Wait; + } self.next_request = request[grow_request::COUNTER].wrapping_add(1); - let count = self.next_count(); - if count == 0 { + if room == 0 { self.refusals += 1; + return Answer::Refuse; + } + Answer::Grant { + first: self.posted, + count: GROW_STEP.min(room).min(self.spare()), } - (self.posted, count) } /// Records `count` blocks as posted after their records were written and returns the new @@ -154,6 +296,37 @@ impl Ledger { } } +/// The engine's idle clock for the reclaim policy: when nothing was published and nothing +/// pending, from the pass that retired the last flight on. Noted at the end of every poll pass +/// and publication (a pass that empties the flights is not re-armed, so a note only at the +/// start of a pass with empty flights depends on a backup pass happening to run) and, busy, +/// when a job is enqueued. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct IdleClock { + /// When the idle began; 0 while busy. + since_ns: u64, +} + +impl IdleClock { + /// Notes the engine idle or busy at `now_ns`; an idle already begun keeps its start. + pub(crate) fn note(&mut self, idle: bool, now_ns: u64) { + if !idle { + self.since_ns = 0; + } else if self.since_ns == 0 { + self.since_ns = now_ns.max(1); + } + } + + /// The idle time at `now_ns`; 0 while busy. + pub(crate) fn idle_ns(&self, now_ns: u64) -> u64 { + if self.since_ns == 0 { + 0 + } else { + now_ns.wrapping_sub(self.since_ns) + } + } +} + /// The halt epoch a request names (u64 at words 5..6). pub(crate) const fn request_epoch(request: &[u32; 14]) -> u64 { request[grow_request::EPOCH] as u64 | (request[grow_request::EPOCH + 1] as u64) << 32 @@ -185,12 +358,19 @@ mod tests { q } + /// A ledger with the heap backed whole. + fn backed(grow: bool, blocks: u32, max_blocks: u32) -> Ledger { + let mut l = Ledger::new(grow, blocks, max_blocks); + l.backed = MAX_BLOCKS; + l + } + #[test] fn clamps() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); assert_eq!( - (l.initial, l.max, l.posted, l.next_request), - (1024, 2048, 1024, 0) + (l.initial, l.max, l.posted, l.backed, l.next_request), + (1024, 2048, 1024, 4096, 0) ); assert_eq!(l.producer(), 1024); assert_eq!(Ledger::new(true, 10, 20).initial, 64); @@ -198,13 +378,14 @@ mod tests { assert_eq!(Ledger::new(true, 5000, 100).initial, 4095); assert_eq!(Ledger::new(true, 512, 1024).max, 1024); assert_eq!(Ledger::new(true, 512, 100).max, 512); + assert_eq!(Ledger::new(true, 512, 100).backed, 0); let fixed = Ledger::new(false, 512, 1024); assert_eq!((fixed.initial, fixed.max, fixed.posted), (4095, 4095, 4095)); - // The values the driver passes (tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS) are kept. - let l = Ledger::new(true, 1024, 4095); + // The values the driver passes (g15_pb_blocks 64, tuning::PB_MAX_BLOCKS) are kept. + let l = Ledger::new(true, 64, 4095); assert_eq!( (l.initial, l.max, l.posted, l.producer()), - (1024, 4095, 1024, 1024) + (64, 4095, 64, 64) ); } @@ -235,18 +416,19 @@ mod tests { for grow in [true, false] { for blocks in values { for max_blocks in values { - let mut l = Ledger::new(grow, blocks, max_blocks); + let mut l = backed(grow, blocks, max_blocks); assert!(MIN_BLOCKS <= l.initial && l.initial <= l.max && l.max <= MAX_POSTED); assert_eq!(l.posted, l.initial); assert!((1..=MAX_POSTED).contains(&l.producer())); // Every producer of a growth to the maximum stays in 1..=4095. let mut counter = 0; loop { - let (first, n) = l.accept(&request(5, counter, 1)); + let (first, n) = match l.accept(&request(5, counter, 1)) { + Answer::Grant { first, count } => (first, count), + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert_eq!(first, l.posted); - if n == 0 { - break; - } let producer = l.post(n); assert!((1..=MAX_POSTED).contains(&producer)); assert_eq!(producer, l.posted); @@ -261,7 +443,7 @@ mod tests { #[test] fn admission() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); let q = request(5, 0, 3); assert!(l.admits(&q, true, true, 3)); assert!(!l.admits(&q, false, true, 3)); @@ -283,17 +465,20 @@ mod tests { #[test] fn growth_to_the_maximum() { // 512 initial and 1024 maximum blocks. - let mut l = Ledger::new(true, 512, 1024); + let mut l = backed(true, 512, 1024); let mut grown = 0; let mut counter = 0; loop { let q = request(5, counter, 1); assert!(l.admits(&q, true, true, 1)); - let (first, n) = l.accept(&q); - assert_eq!(first, 512 + grown); - if n == 0 { - break; - } + let n = match l.accept(&q) { + Answer::Grant { first, count } => { + assert_eq!(first, 512 + grown); + count + } + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert!(n <= 10); assert_eq!(l.post(n), (512 + grown + n) % 4096); grown += n; @@ -310,26 +495,97 @@ mod tests { assert_eq!(l.refusals, 1); assert!(!l.exhausted()); let q = request(5, counter + 3, 1); - assert_eq!(l.accept(&q), (1024, 0)); + assert_eq!(l.accept(&q), Answer::Refuse); assert!(l.exhausted()); assert!(!l.admits(&request(5, counter + 4, 1), true, true, 1)); - // A fresh ledger starts over. + // A fresh ledger starts over, and so does a re-initialized one (backing kept). assert!(!Ledger::new(true, 512, 1024).exhausted()); + l.reinit(); + assert_eq!( + (l.posted, l.next_request, l.refusals, l.backed), + (512, 0, 0, 4096) + ); + assert!(!l.exhausted() && !l.refusal_logged); } - /// Backing for every ledger the constructor makes, the driver's included: the chunks from - /// the initial one are contiguous, at most [`CHUNK_BLOCKS`] each, end at the maximum and - /// number at most [`max_chunks`]; growth with the backing made first never posts a block - /// past it. #[test] - fn backing_chunks() { - assert_eq!(max_chunks(1024, 4095), 1 + 96); - assert_eq!(max_chunks(MIN_BLOCKS, MAX_POSTED), 1 + 126); - assert_eq!(max_chunks(MAX_POSTED, MAX_POSTED), 1); - assert_eq!(next_chunk(1024, 4095), Some((1024, 32))); - assert_eq!(next_chunk(4064, 4095), Some((4064, 31))); - assert_eq!(next_chunk(4095, 4095), None); - assert_eq!(next_chunk(4096, 4095), None); + fn producer_wraps_at_k() { + let mut l = backed(true, 2048, 2048); + assert_eq!(l.producer(), 2048); + l.posted = 4095; + assert_eq!(l.post(1), 0); + assert_eq!(l.post(10), 10); + } + + #[test] + fn backing() { + // 64 initial blocks: two chunks (the initial posting and the reserve). + assert_eq!(Ledger::chunks_for(64), 2); + assert_eq!(Ledger::chunks_for(1024), 9); + assert_eq!(Ledger::chunks_for(4095), MAX_CHUNKS); + assert_eq!(MAX_CHUNKS, 32); + let mut l = Ledger::new(true, 64, 4095); + assert_eq!(l.keep_chunks(), 2); + assert_eq!(l.spare(), 0); + assert!(l.refill_due()); + // Nothing backed: a request waits and the counter does not advance. + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + l.back_chunk(); + l.back_chunk(); + assert_eq!((l.backed, l.spare()), (256, 192)); + assert!(!l.refill_due()); + // Granting until the reserve line, then a refill is due; at the end of the backing the + // grant is clipped and the next request waits. + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 10 + } + ); + assert_eq!(l.post(10), 74); + assert_eq!(l.next_request, 1); + l.posted = 128; + assert!(!l.refill_due()); + l.posted = 129; + assert!(l.refill_due()); + l.posted = 250; + assert_eq!( + l.accept(&request(3, 1, 1)), + Answer::Grant { + first: 250, + count: 6 + } + ); + l.post(6); + assert_eq!(l.accept(&request(3, 2, 1)), Answer::Wait); + // The heap end: no refill past the whole heap, the maximum refuses. + l.backed = MAX_BLOCKS; + l.posted = 4090; + assert!(!l.refill_due()); + assert_eq!( + l.accept(&request(3, 2, 1)), + Answer::Grant { + first: 4090, + count: 5 + } + ); + l.post(5); + assert_eq!(l.accept(&request(3, 3, 1)), Answer::Refuse); + // Re-initialization keeps the backing; dropping it to the kept chunks. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 4096)); + l.unback_to(l.keep_chunks()); + assert_eq!(l.backed, 256); + } + + /// Every ledger the constructor makes, the driver's included, backed a chunk at a time + /// whenever the reserve is short: no grant posts a block past the backing, no request waits, + /// and the backing ends within the heap. + #[test] + fn backing_ahead_of_every_grant() { let values = [ 0, 1, MIN_BLOCKS, 65, 1023, 1024, 1025, 3072, 4094, MAX_POSTED, 5000, ]; @@ -337,40 +593,308 @@ mod tests { for blocks in values { for max_blocks in values { let mut l = Ledger::new(grow, blocks, max_blocks); - let (mut backed, mut chunks) = (l.initial, 1); + for _ in 0..Ledger::chunks_for(l.initial) { + l.back_chunk(); + } let mut counter = 0; loop { - let end = l.posted + l.next_count(); - while backed < end { - let (first, n) = next_chunk(backed, l.max).unwrap(); - assert_eq!(first, backed); - assert!(n >= 1 && n <= CHUNK_BLOCKS); - backed += n; - chunks += 1; - } - let (first, n) = l.accept(&request(5, counter, 1)); - assert!(first + n <= backed); - if n == 0 { - break; + match l.accept(&request(5, counter, 1)) { + Answer::Grant { first, count } => { + assert_eq!(first, l.posted); + assert!(count >= 1 && first + count <= l.backed); + l.post(count); + } + Answer::Refuse => break, + Answer::Wait => panic!("waited at {} of {}", l.posted, l.backed), } - l.post(n); counter += 1; + while l.refill_due() { + l.back_chunk(); + } } - assert_eq!((l.posted, backed), (l.max, l.max)); - assert!(chunks <= max_chunks(l.initial, l.max)); - assert_eq!(next_chunk(backed, l.max), None); + assert_eq!(l.posted, l.max); + assert!(l.max <= l.backed && l.backed <= MAX_BLOCKS); } } } } #[test] - fn producer_wraps_at_k() { - let mut l = Ledger::new(true, 2048, 2048); - assert_eq!(l.producer(), 2048); - l.posted = 4095; - assert_eq!(l.post(1), 0); - assert_eq!(l.post(10), 10); + fn reclaim() { + let mut l = Ledger::new(true, 64, 4095); + let s = 1_000_000_000; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1024; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1025; + assert!(l.reclaim_due(1024, s, s)); + assert!(!l.reclaim_due(1024, s - 1, s)); + // The clock: busy is 0, an idle keeps its start, busy ends it. + let mut c = IdleClock::default(); + assert_eq!(c.idle_ns(10 * s), 0); + c.note(true, 10 * s); + c.note(true, 12 * s); + assert_eq!(c.idle_ns(15 * s), 5 * s); + c.note(false, 16 * s); + assert_eq!(c.idle_ns(20 * s), 0); + // A clock started at 0 ns still counts as idle. + c.note(true, 0); + assert_eq!(c.idle_ns(s), s - 1); + // Nothing grown since the initialization: no re-initialization, whatever the floor. + let l = Ledger::new(true, 64, 4095); + assert!( + !l.reclaim_due(32, 5 * s, s), + "re-initialization with nothing grown" + ); + } + + /// Heavy scenes of ~1420 blocks each, new clients back to + /// back. The engine re-initializes at a client boundary once past half the maximum, so the + /// heap never reaches it; the backing stays (no refill on the next scene). + #[test] + fn back_to_back_heavy_scenes() { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + let mut counter = 0; + for _ in 0..10 { + if l.drain_reinit_due(DRAIN_IDLE_NS) { + l.reinit(); + counter = l.next_request; + } + let start = l.posted; + while l.posted < start + 1420 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + assert!(l.posted < l.max); + } + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + assert_eq!(l.backed, 24 * CHUNK_BLOCKS); + l.posted = 2047; + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + l.posted = 2048; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // Not between two renders of one client. + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS - 1)); + assert!(!l.drain_reinit_due(0)); + // An initial posting above half the maximum: nothing grew, nothing to drain. + let l = Ledger::new(true, 3000, 4095); + assert!( + !l.drain_reinit_due(DRAIN_IDLE_NS), + "drain with nothing grown" + ); + } + + /// A client-boundary re-initialization keeps the backing + /// of a heavy scene, the next client posts little; once idle, the backing above what the + /// posted blocks need is due to be freed although the posted count is below the floor. + #[test] + fn kept_backing_freed_at_idle() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(!l.free_due(5 * s, s)); + l.reinit(); + l.posted = 264; + assert!(!l.reclaim_due(1024, 5 * s, s)); + assert!(!l.free_due(s - 1, s)); + assert!(l.free_due(s, s)); + // After the re-initialization and the free, nothing more is due. + l.reinit(); + l.unback_to(l.keep_chunks()); + assert!(!l.free_due(5 * s, s)); + // A heap backed for what it posts plus the reserve is not freed. + l.posted = 1000; + l.unback_to(Ledger::chunks_for(1000)); + assert!(!l.free_due(5 * s, s)); + } + + /// A heavy scene, a reclaim that frees the backing, refills during the next heavy scene, + /// an idle gap with a small job in it, and the next render must reclaim again. + #[test] + fn reclaim_after_free_then_refill() { + let s = 1_000_000_000; + let (floor, min_idle) = (1024, s); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + let mut clock = IdleClock::default(); + // A heavy scene grows the PB to 1484 blocks, refilling as it goes. + let heavy = |l: &mut Ledger, clock: &mut IdleClock, t: u64| { + clock.note(false, t); + let mut counter = l.next_request; + while l.posted < 1484 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + // The pass that retires the last flight notes the idle; no later pass may run. + clock.note(true, t + s); + }; + heavy(&mut l, &mut clock, 100 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // 5 s later a small job arrives: the reclaim is due; it frees the backing above the + // kept chunks; the re-initialization happens at the next render. + let t = 106 * s; + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + l.unback_to(l.keep_chunks()); + clock.note(false, t); + assert_eq!(l.backed, 2 * CHUNK_BLOCKS); + clock.note(true, t + s / 2); + // The next render re-initializes, then the second heavy scene refills to 1484 again. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 256)); + heavy(&mut l, &mut clock, 110 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // The same gap again: the reclaim must be due at the small job's arrival. + let t = 116 * s; + assert_eq!(clock.idle_ns(t), 5 * s); + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + } + + /// The delay names exactly the idle time at which the reclaim check first acts: nothing is + /// due one nanosecond before it, something is due at it, and nothing ever becomes due + /// when it is `None`. + #[test] + fn reclaim_delay_matches_the_checks() { + let s = 1_000_000_000; + let due = |l: &Ledger, level: u32, floor: u32, idle: u64, min_idle: u64| { + level != 0 + && (l.reclaim_due(floor, idle, min_idle) + || (level >= 2 && l.free_due(idle, min_idle)) + || l.drain_reinit_due(idle)) + }; + for level in 0..=2 { + for floor in [0, 64, 1024, 4095] { + for min_idle in [DRAIN_IDLE_NS / 2, 100_000_000, s] { + for posted in [64, 200, 1024, 1025, 2047, 2048, 2900, 4095] { + for chunks in [2, 3, 9, 23, MAX_CHUNKS] { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(chunks); + l.posted = posted; + match l.reclaim_delay(level, floor, 0, min_idle) { + None => { + for idle in [0, DRAIN_IDLE_NS, min_idle, 100 * s] { + assert!(!due(&l, level, floor, idle, min_idle)); + } + } + Some(d) => { + assert!(due(&l, level, floor, d, min_idle)); + assert!(d == 0 || !due(&l, level, floor, d - 1, min_idle)); + // Counted from an idle already begun. + assert_eq!( + l.reclaim_delay(level, floor, d / 2, min_idle), + Some(d - d / 2) + ); + assert_eq!( + l.reclaim_delay(level, floor, d + s, min_idle), + Some(0) + ); + } + } + } + } + } + } + } + // The case of the J516C run: a client-boundary re-initialization left 23 chunks for + // 264 posted blocks; the free is due after g15_pb_idle_ms of idle. + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 264; + assert_eq!(l.reclaim_delay(2, 1024, 0, s), Some(s)); + assert_eq!(l.reclaim_delay(1, 1024, 0, s), None); + assert_eq!(l.reclaim_delay(0, 1024, 0, s), None); + // A desktop at the initial posting with the kept chunks: nothing to wait for. + l.unback_to(l.keep_chunks()); + l.posted = 64; + assert_eq!(l.reclaim_delay(2, 1024, 0, s), None); + // Nothing grown, with an initial posting above half the maximum or above the floor: + // nothing armed. + let mut l = Ledger::new(true, 3000, 4095); + l.unback_to(Ledger::chunks_for(3000)); + assert_eq!( + l.reclaim_delay(2, 1024, 0, s), + None, + "armed with nothing grown" + ); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert_eq!( + l.reclaim_delay(2, 32, 0, s), + None, + "armed with nothing grown" + ); + } + + /// A client-boundary re-initialization with 23 chunks backed and no render after it: the + /// kept backing is due to be freed after the long idle, measured from the start of the idle, + /// and once freed nothing more is due. + #[test] + fn pending_reinit_frees_kept_backing() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // The poll after the 50 ms re-initialization waits for the rest of the long idle. + assert_eq!( + l.pending_free_delay(2, DRAIN_IDLE_NS, s), + Some(s - DRAIN_IDLE_NS) + ); + assert!(!l.pending_free_due(s - 1, s)); + assert!(l.pending_free_due(s, s)); + assert_eq!(l.pending_free_delay(2, s + 1, s), Some(0)); + assert_eq!(l.pending_free_delay(1, 0, s), None); + assert_eq!(l.pending_free_delay(0, 0, s), None); + l.unback_to(l.keep_chunks()); + assert!(!l.pending_free_due(100 * s, s)); + assert_eq!(l.pending_free_delay(2, 0, s), None); + } + + #[test] + fn grants_only_backed_blocks() { + let mut l = Ledger::new(true, 64, 4095); + // Nothing backed above the initial posting: a request waits and keeps its counter. + l.backed = 64; + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + // A grant is clipped to the backed blocks; the next request waits again. + l.backed = 70; + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 6 + } + ); + l.post(6); + assert_eq!(l.next_request, 1); + assert_eq!(l.accept(&request(3, 1, 1)), Answer::Wait); + // At the maximum the refusal does not wait for backing. + l.posted = l.max; + l.backed = l.max; + assert_eq!(l.accept(&request(3, 1, 1)), Answer::Refuse); + // A re-initialization keeps the backing. + l.reinit(); + assert_eq!((l.posted, l.backed, l.next_request), (64, 4095, 0)); } #[test] diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index 41cc7e438a6d65..e2b9d884bc25b8 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -10,8 +10,9 @@ //! ([`object::Objects::clients`]) as driver mappings at their fixed GVAs, and reserve //! [`super::hw::client::CLIENT_APERTURE`], the pass-slot range and the PB page list //! (`vm.install_driver_mappings`): user binds there fail (EINVAL). The aperture holds the -//! whole PB range, whose backing is mapped later, before the VM's first prefix is published -//! (`Firmware::map_pb`, counted in [`VmState::pb_chunks`]); +//! whole PB range, whose backing chunks are mapped later, before the VM's prefixes are +//! published (`Firmware::map_pb`) and, for the VM of the flights, when a chunk is backed +//! ([`VmState::map_heap`], [`VmState::map_chunk`]); //! - bind it to an exclusive TTBAT slot ([`mmu::Uat::bind`]) for as long as its [`VmState`] //! lives; at most [`super::hw::sizes::MAX_VMS`] such VMs (ENOSPC beyond, checked before //! binding so the slot allocator never blocks). @@ -29,7 +30,7 @@ //! alike. use core::ops::Range; -use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicU32, Ordering}; use kernel::{ drm::gem::BaseObject, new_mutex, @@ -104,6 +105,20 @@ impl ClientAlias { } } +/// A PB heap chunk mapped in a VM at its fixed GVA ([`super::pools::chunk_gva`]). +struct HeapMapping { + gem: ARef, + _mapping: mmu::KernelMapping, +} + +/// The PB heap chunks mapped in a VM, in heap order (chunk `i` at `chunk_gva(i)`). Changed +/// under the device lock (refill, publication). +#[pin_data] +struct HeapMappings { + #[pin] + list: Mutex>, +} + /// G15 state of one user VM, shared by the registry, its queues and its jobs. pub(crate) struct VmState { /// The `mmu::Vm` identifier (`Gpu::ids.vm`). @@ -117,9 +132,8 @@ pub(crate) struct VmState { /// The exclusive slot binding, released before the live count drops. bind: Option, registry: Arc, - /// PB backing chunks mapped into the VM, from the first (`Firmware::map_pb`); changed only - /// under the device lock. - pub(crate) pb_chunks: AtomicUsize, + /// The PB heap chunks mapped in this VM. + heap: Arc, } impl VmState { @@ -127,6 +141,62 @@ impl VmState { pub(crate) fn prune_registry(&self) { self.registry.prune(); } + + /// Maps PB heap chunk `i`, the next one this VM lacks, at its GVA. The alias is held only + /// once it is in the list. + fn map_one(&self, heap: &mut KVec, i: usize, gem: ARef) -> Result { + if i != heap.len() { + return Err(EINVAL); + } + heap.reserve(1, GFP_KERNEL)?; + let mapping = self.vm.map_at( + super::pools::chunk_gva(i), + super::pools::CHUNK_BYTES, + &gem, + mmu::Prot::from_pte(hw::pte::LEAF_CACHED), + false, + )?; + heap.push_within_capacity(HeapMapping { + gem, + _mapping: mapping, + }) + .map_err(|_| ENOMEM) + } + + /// Maps the backed chunks (`gems`, in heap order) this VM does not hold yet. A chunk whose + /// object changed since (freed and backed again) is unmapped first, with everything above + /// it, and so are chunks past the backing. + pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result { + let mut heap = self.heap.list.lock(); + let mut n = 0; + for (i, gem) in gems.enumerate() { + n = i + 1; + if let Some(have) = heap.get(i) { + if core::ptr::eq(&*have.gem, &*gem) { + continue; + } + heap.truncate(i); + } + self.map_one(&mut heap, i, gem)?; + } + heap.truncate(n); + Ok(()) + } + + /// Maps the newly backed chunk `i` into this VM (the VM of a flight, at a refill). + pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result { + let mut heap = self.heap.list.lock(); + if i < heap.len() { + // Already mapped (a flight of the same VM came first). + return Ok(()); + } + self.map_one(&mut heap, i, gem) + } + + /// Unmaps the heap chunks from index `keep` on. + pub(crate) fn unmap_heap_above(&self, keep: usize) { + self.heap.list.lock().truncate(keep); + } } impl Drop for VmState { @@ -251,6 +321,17 @@ impl Registry { .ok_or(ENOENT) } + /// The VMs whose client handle is open (and the closed ones not pruned yet), for a walk + /// that takes VM locks: not under the registry lock. + pub(crate) fn open(&self) -> Result>> { + let vms = self.vms.lock(); + let mut out = KVec::with_capacity(vms.len(), GFP_KERNEL)?; + for state in vms.iter() { + out.push(state.clone(), GFP_KERNEL)?; + } + Ok(out) + } + /// Latches `error` on every VM whose client handle is open. The fail-stop path calls it under /// the device lock. pub(crate) fn fail_all(&self, error: Error) { @@ -292,6 +373,13 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { let status = vm.status().ok_or(EINVAL)?.clone(); let (mappings, reserved) = Registry::driver_mappings(&vm, aliases)?; vm.install_driver_mappings(mappings, reserved)?; + // No PB chunk yet: they are mapped before the VM's first prefix (`Firmware::map_pb`). + let heap = Arc::pin_init( + pin_init!(HeapMappings { + list <- new_mutex!(KVec::new(), "g15::HeapMappings::list"), + }), + GFP_KERNEL, + )?; // The context names the root only now: everything above is already visible to the walker. let bind = gpu.uat.bind(&vm)?; let slot = bind.slot(); @@ -305,7 +393,7 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { vm: vm.clone(), bind: Some(bind), registry: registry.clone(), - pb_chunks: AtomicUsize::new(0), + heap, }, GFP_KERNEL, )?; diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 7cdedc9f0cc951..e7f350d5c7f877 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -3,16 +3,18 @@ """Run the G15 PB growth answer and its on-demand backing, actual bodies, against a model. The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), -answer_request(), map_pb(), running_vm() and back_pb(), Pools::soft_max() and backed(), -PbChunk and Pools::new's PB backing statements; pb_max_blocks() from params.rs; the PB sizes, -tuning and PB leaf word from hw.rs; Prot::from_pte() and is_cached_noncoherent() from -pgtable.rs; and the parameter's default from asahi_core.rs. It includes pools/growth.rs, -fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, -the GEM backing allocation, the shared layer's map_at() and append_driver_mappings(), and the -DEVCTL wait, which logs every allocation, alias map, append, unmap, free, block record, -barrier, producer store, reply and log line in order. Allocation, alias map and append -failures can be injected. Source checks: publish_prefix_into() starts with the PB catch-up, -and pools.rs asserts the PB leaf GPU-only at compile time. +answer_request(), map_pb(), refill_backing(), note_idle(), reclaim_check(), free_backing() and +reclaim_delay_ns(), Pools::soft_max(), map_heap(), free_chunks() and back_chunk(), PbChunk, +chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from vm.rs VmState::map_one(), +map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks(), pb_blocks(), pb_reclaim(), +pb_idle_ns() and pb_reclaim_blocks() from params.rs; the PB sizes, tuning and PB leaf word from +hw.rs; Prot::from_pte() and is_cached_noncoherent() from pgtable.rs; and the parameters' defaults +from asahi_core.rs. It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies +run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() +and the DEVCTL wait, which logs every allocation, alias map, unmap, free, block record, barrier, +producer store, reply and log line in order. Allocation and alias map failures can be injected. +Source checks: publish_prefix_into() starts with the PB catch-up, and pools.rs asserts the PB leaf +GPU-only at compile time. What it shows: - a poll pass answers only the requests queued when it starts, in order; one the reply's wait @@ -21,22 +23,42 @@ offset, each producer store follows its records and a barrier, each reply follows its producer store and a barrier, and every producer is in 1..=4095, ending at 4095; - at the maximum two refusals are answered; further requests get no reply, count as bad and - warn once until the ledger is made again, as init() does; + warn once until the ledger is reset, as init() does; - the soft maximum is the parameter clamped to 64..=4095 and then to the ledger's initial..=max; growth answers above it are logged at info level, and the one that reaches it exactly is not; - invalid requests (type, slot, stale counter, epoch, no live render) get no reply, and requests rejected so allocate nothing; -- the backing: the initial blocks only when the GPU starts; chunks of at most 4 MiB, allocated - and held by the running VM before any record or producer names their blocks, never past the - maximum, at most growth::max_chunks(); a failed allocation, alias map or append posts and - answers nothing, leaves the request counter and the VM's count alone and drops what is not - held, and the next request is answered; a failed reply keeps everything; -- a VM's count always equals the aliases it holds, contiguous from the heap's start; a VM that - runs after another grew the backing maps it first (at publication, and again before it grows); - a partial catch-up counts only held aliases; -- the one-VM rule: a catch-up with a flight of another VM, or a count past the chunks, and a - grant with flights of two VMs, are refused (EPROTO) before anything is mapped or posted; +- the backing: the chunks of the initial posting and one reserve chunk when the GPU starts; + chunks of 16 MiB, backed after the replies of a pass (never between them) while fewer than a + reserve chunk remain, mapped into the VM of the flights before any record or producer names + their blocks, never past the heap; with the reserve dry a request waits at the head of the + queue, keeps its counter and is answered once a chunk is backed; a chunk that cannot be mapped + into the running VM is dropped again; a pending re-initialization backs nothing while no + render is published, and still backs for a render published before it; a failed reply keeps + everything; +- a VM's aliases are the chunks, contiguous from the heap's start; a VM that runs after another + grew the backing maps it first (at publication); a partial catch-up keeps the aliases made; a + chunk freed and backed again (an idle reclaim) is unmapped and mapped again at publication; +- the one-VM rule: a catch-up with a flight of another VM is refused (EPROTO) before anything + is mapped; +- the idle reclaim, actual reclaim_check(), free_backing() and note_idle() over a clock: a heavy + client that exits with nothing rendered after it is re-initialized after 50 ms with the backing + kept, and the kept backing is freed after the long idle, its posted count kept until the InitBM; + at reclaim level 1 the backing stays; a recovery's pending re-initialization frees nothing; + driven as the idle poll work arms it (reclaim_delay_ns()), the same exit wakes at 50 ms and at + the long idle and arms nothing after; an idle reclaim arms nothing once its re-initialization + is pending; +- nothing grown since the initialization (an initial posting above half the maximum, or a + reclaim floor below it): no re-initialization and nothing armed; +- the actual reclaim keeps the posted count until the next render's InitBM (a model of the + publication: map_pb() first, then for a render prefix with a re-initialization pending the + ledger reset of Pools::init()): the same client rendering again after 60 ms re-posts the + initial blocks and grows again from the kept backing with nothing allocated; after 1.1 s it + grows again over freshly backed chunks, each mapped before it is posted; a compute-only + submission between the reclaim and the InitBM maps, backs and posts nothing; +- a reclaim whose walk of the VMs fails counts nothing freed; the VM's next publication drops + its stale aliases and releases the pages; - the ledger reset of Pools::init() re-posts from the backing without allocating; - the PB leaf decodes as GPU-only, not cached-noncoherent: dropping an alias is a plain unmap; - Ledger::new() is Ledger::new() as of dba2a110de00 (the carried upstream constructor), capped at @@ -47,8 +69,7 @@ refusal or an unanswered request. The model's VMs have no open/closed state: that a closed VM whose jobs still run keeps its aliases comes from the VM lifetime source (Job and VmJobGuard hold the VM), not from these tests. The ledger reset is not a recovery. The shared layer's -mapping and its rollback are modelled, not run. Running only running_vm()'s or only map_pb()'s -one-VM check on a grant is not distinguishable: each covers it. +mapping is modelled, not run. With --mutations, each mutation below must fail its intended test with the expected message, and the control (a driver maximum of the whole 4096-block heap) must pass, held by the ledger's cap. @@ -73,6 +94,7 @@ HW = f"{G15}/hw.rs" CORE = "drivers/gpu/drm/asahi/asahi_core.rs" ENGINE = f"{G15}/engine.rs" +VM = f"{G15}/vm.rs" PGTABLE = "drivers/gpu/drm/asahi/pgtable.rs" OLD_REF = "dba2a110de00743eeecd41edf23e54ede4491d5e" @@ -95,14 +117,16 @@ } }''' -# Pools::new's PB backing, run verbatim by the model. -POOLS_NEW_BACKING = (' // Backing for the initial blocks; growth adds the rest.\n', ' chunks.push(') +# Pools::new's PB backing, run verbatim by the model (up to its log line). +POOLS_NEW_BACKING = (' // Backing for the initial blocks and the reserve; growth adds the rest.\n', + ' dev_info!(\n') # The compile-time proof that a PB alias drop is a plain unmap. PB_LEAF_ASSERT = 'static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent());' -# The two statements of Pools::init() that start a new ledger. -LEDGER_RESET = ['self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS);', - 'self.exhausted_logged = false;'] +# The two statements of Pools::init() that reset the ledger. +LEDGER_RESET = ['self.ledger.reinit();', + 'self.exhausted_logged = false;', + 'self.reclaim_pending = false;'] MUTATIONS = { # Answer until the queue is empty, including requests queued by the replies' waits. @@ -110,9 +134,9 @@ # Leave the last request queued before the pass for later. 'skip-last-queued': (POOLS, 'for _ in 0..queued {', 'for _ in 0..queued.saturating_sub(1) {'), # End the pass at a request that cannot be processed. - 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', - ' self.pools.stats.bad += 1;\n return Ok(());\n' - ' dev_err!('), + 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', + ' self.pools.stats.bad += 1;\n return Ok(());\n' + ' dev_err!('), # Answer three refusals at the maximum. 'three-refusals': (GROWTH, 'pub(crate) const MAX_REFUSALS: u32 = 2;', 'pub(crate) const MAX_REFUSALS: u32 = 3;'), # Answer refusals for ever. @@ -138,42 +162,115 @@ 'let nb = clamp(blocks, MIN_BLOCKS, MAX_BLOCKS);\n' ' (nb, clamp(max_blocks, nb, MAX_BLOCKS))\n' ' } else {\n (MAX_BLOCKS, MAX_BLOCKS)'), - # Post blocks without backing them first. - 'grant-without-backing': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', - ' let (first, n) = self.pools.ledger.accept(request);'), - # Back the blocks only after the request is accepted. - 'back-after-accept': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', - ' let (first, n) = self.pools.ledger.accept(request);\n self.back_pb(end, ctx)?;'), - # Count a catch-up alias before the VM holds it. - 'catch-up-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n mapped += 1;\n' - ' vm.pb_chunks.store(mapped, Ordering::Relaxed);', - ' mapped += 1;\n vm.pb_chunks.store(mapped, Ordering::Relaxed);\n' - ' vm.vm.append_driver_mappings(aliases)?;'), - # Count a growth chunk's alias before the VM holds it. - 'growth-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.', - ' vm.pb_chunks.store(self.pools.chunks.len() + 1, Ordering::Relaxed);\n' - ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.'), - # Keep a chunk whose alias the VM never held. - 'keep-unappended-chunk': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.\n' - ' self.pools\n .chunks\n .push_within_capacity(chunk)\n' - ' .map_err(|_| EIO)?;', - ' // Within the capacity checked above.\n self.pools\n .chunks\n' - ' .push_within_capacity(chunk)\n .map_err(|_| EIO)?;\n' - ' vm.vm.append_driver_mappings(aliases)?;'), - # Grow without catching the running VM up first. - 'no-growth-catch-up': (POOLS, ' let vm = self.running_vm(ctx)?;\n self.map_pb(&vm)?;\n', - ' let vm = self.running_vm(ctx)?;\n'), + # Grant without the backing: neither wait nor clip. + 'grant-without-backing': [(GROWTH, ' if room != 0 && self.spare() == 0 {\n return Answer::Wait;\n }\n', ''), + (GROWTH, 'count: GROW_STEP.min(room).min(self.spare()),', 'count: GROW_STEP.min(room),')], + # Allocate before the pass's answers. + 'refill-before-answers': [(POOLS, ' self.refill_backing();\n Ok(())', ' Ok(())'), + (POOLS, ' let queued = self.events.grow.len();\n', + ' self.refill_backing();\n let queued = self.events.grow.len();\n')], + # Back a chunk without mapping it into the running VM. + 'refill-maps-no-flight': (POOLS, ' flight.job.vm.map_chunk(i, gem.clone())?;\n', + ' let _ = (flight, i, gem.clone());\n'), + # Drop a request that waits for backing. + 'no-wait-requeue': (POOLS, ' if self.events.grow.insert_within_capacity(0, request).is_err() {\n' + ' self.pools.stats.bad += 1;\n }\n', ''), + # Keep a chunk that the running VM could not map. + 'refill-failure-keeps-chunk': (POOLS, ' self.pools.chunks.truncate(backed);\n', ''), + # Back chunks while a re-initialization is pending. + 'refill-during-reinit': (POOLS, ' if self.recovery.reinit && !render {\n return;\n }\n', ''), + # Skip the refill on any pending re-initialization, also with a live render (a stall). + 'refill-skip-with-live-render': (POOLS, 'if self.recovery.reinit && !render {', 'if self.recovery.reinit {'), + # Refill for any live flight, also compute-only ones that cannot ask for growth. + 'refill-for-any-flight': (POOLS, 'if self.recovery.reinit && !render {', + 'if self.recovery.reinit && self.engine.flights.is_empty() {'), + # Keep a VM's alias of a chunk that was freed and backed again. + 'map-without-identity': (VM, ' if core::ptr::eq(&*have.gem, &*gem) {', ' if true {'), + # Stop the reclaim check on a reclaim's pending re-initialization (the kept backing stays). + 'pending-reinit-stops-free': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + # Free the kept backing at reclaim level 1. + 'pending-free-at-level-1': (POOLS, 'if level >= 2 && ledger.pending_free_due(idle_ns, min_idle) {', + 'if ledger.pending_free_due(idle_ns, min_idle) {'), + # Do not record that the pending re-initialization is a reclaim's. + 'reclaim-pending-not-set': (POOLS, ' self.pools.reclaim_pending = true;\n', ''), + # Free for any pending re-initialization, also a recovery's. + 'free-on-recovery-reinit': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + # Both checks stop on any pending re-initialization: the code before the kept-backing free. + 'old-reinit-guards': [(POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n')], + # Arm nothing while a reclaim's re-initialization is pending. + 'pending-reinit-arms-nothing': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n'), + # Arm the poll for the expired 50 ms instead of the rest of the long idle. + 'pending-delay-from-drain': (GROWTH, 'Some(min_idle_ns.saturating_sub(idle_ns))', + 'Some(DRAIN_IDLE_NS.saturating_sub(idle_ns))'), + # Re-initialize at an idle with nothing grown (posted above the floor only). + 'reclaim-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns', + 'self.posted > floor && idle_ns >= min_idle_ns'), + # Re-initialize at a client boundary with nothing grown (initial above half the maximum). + 'drain-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS', + 'self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS'), + # Arm the poll with nothing grown. + 'delay-nothing-grown': (GROWTH, ' let grown = self.posted > self.initial;\n', ' let grown = true;\n'), + # Reset the posted count at the idle reclaim instead of at the next render's InitBM. + 'reinit-at-idle-reclaim': (POOLS, ' if level >= 2 && idle {\n freed = self.free_backing();\n', + ' if level >= 2 && idle {\n self.pools.ledger.reinit();\n freed = self.free_backing();\n'), + # Reset the posted count at any reclaim instead of at the next render's InitBM. + 'reinit-at-any-reclaim': (POOLS, ' self.pools.reclaim_pending = true;\n', + ' self.pools.reclaim_pending = true;\n self.pools.ledger.reinit();\n'), + # Count the chunks freed although the VMs could not be walked. + 'freed-without-walk': (POOLS, ' );\n 0\n }\n };\n', + ' );\n dropped\n }\n };\n'), # Catch up without the one-VM check. - 'no-activation-guard': (POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), - # Neither the catch-up's nor the grant's one-VM check (each alone covers a grant). - 'no-same-vm-checks': [(POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), - (POOLS, 'if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) {', - 'if vm.slot != ctx {')], + 'no-activation-guard': (POOLS, '.filter(|f| !core::ptr::eq(&*f.job.vm, vm))', '.filter(|_| false)'), # Back the whole heap when the GPU starts. - 'initial-backs-whole-heap': (POOLS, 'chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?;', - 'chunks.push(PbChunk::new(dev, 0, buffer::MAX_BLOCKS)?, GFP_KERNEL)?;'), - # Growth chunks of 8 MiB. - 'chunks-above-4mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 32;', 'pub(crate) const CHUNK_BLOCKS: u32 = 64;'), + 'initial-backs-whole-heap': (POOLS, 'for _ in 0..Ledger::chunks_for(ledger.initial) {', + 'for _ in 0..growth::MAX_CHUNKS {'), + # Growth chunks of 32 MiB. + 'chunks-above-16mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 128;', 'pub(crate) const CHUNK_BLOCKS: u32 = 256;'), # A firmware-visible cached PB leaf (its drop would ask the firmware to flush). 'pb-leaf-fw-visible': (HW, 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c83;', 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c03;'), @@ -181,28 +278,43 @@ # The test each mutation must fail, and a piece of its failure message. INTENDED = { - 'drain-until-empty': ('queued_snapshot_keeps_order', 'unbounded answering in one poll pass'), + 'drain-until-empty': ('queued_snapshot_keeps_order', 'replies in the first pass'), 'skip-last-queued': ('queued_snapshot_keeps_order', 'replies in the first pass'), 'return-after-request-error': ('unprocessed_request_does_not_end_the_pass', 'requests taken in the pass'), 'three-refusals': ('default_growth_to_exhaustion', 'want Some(Log("warn"'), 'never-exhausted': ('default_growth_to_exhaustion', 'growth did not end'), 'warn-every-time': ('default_growth_to_exhaustion', 'later request after exhaustion'), - 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(1034)), want Some(Sync)'), + 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(74)), want Some(Sync)'), 'no-reply-barrier': ('default_growth_to_exhaustion', 'got Some(Reply([7, 1, 0, 5, 0, 256, 3, 1, 0, 0, 0, 0, 0, 0])), want Some(Sync)'), 'soft-boundary-inclusive': ('soft_max_boundaries', 'got Some(Log("info", "")), want Some(Log("dbg", ""))'), 'soft-max-unclamped': ('soft_max_bounds', 'soft_max, param 0'), 'uncapped-ledger': ('constructor_is_the_old_one_capped', 'new(true, '), - 'grant-without-backing': ('growth_backs_before_posting', 'record of block'), - 'back-after-accept': ('allocation_failure_posts_nothing', 'ledger after a backing failure'), - 'catch-up-cursor-before-append': ('partial_catch_up_counts_only_held_aliases', 'count after a failed append'), - 'growth-cursor-before-append': ('append_failure_posts_nothing', 'count differs from held aliases'), - 'keep-unappended-chunk': ('append_failure_posts_nothing', 'chunks after a backing failure'), - 'no-growth-catch-up': ('growth_catches_the_running_vm_up_first', 'running VM not caught up'), + 'grant-without-backing': ('dry_reserve_waits_and_keeps_order', 'record of block'), + 'refill-before-answers': ('allocation_follows_the_replies', 'allocation before a reply'), + 'refill-maps-no-flight': ('growth_backs_before_posting', 'record of block'), + 'no-wait-requeue': ('dry_reserve_waits_and_keeps_order', 'waiting request lost'), + 'refill-failure-keeps-chunk': ('refill_map_failure_drops_the_chunk', 'chunk kept after a failed map'), + 'refill-during-reinit': ('pending_reinit_without_flights_backs_nothing', 'backed during a pending re-initialization'), + 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), + 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), + 'map-without-identity': ('freed_chunks_are_mapped_again', 'alias of another object'), + 'pending-reinit-stops-free': ('client_boundary_reinit_arms_the_long_idle', 'poll woke in a loop'), + 'old-reinit-guards': ('client_boundary_reinit_arms_the_long_idle', 'kept backing not freed'), + 'pending-reinit-arms-nothing': ('client_boundary_reinit_arms_the_long_idle', 'kept backing not freed'), + 'pending-delay-from-drain': ('client_boundary_reinit_arms_the_long_idle', 'poll woke in a loop'), + 'pending-free-at-level-1': ('client_boundary_reinit_at_level_1_keeps_the_backing', 'freed at level 1'), + 'reclaim-pending-not-set': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), + 'free-on-recovery-reinit': ('recovery_reinit_frees_nothing', 'freed for a recovery'), + 'reclaim-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (64 blocks, floor 32)'), + 'drain-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (3000 blocks'), + 'delay-nothing-grown': ('nothing_grown_arms_nothing', 'poll woke in a loop'), + 'reinit-at-idle-reclaim': ('same_client_render_after_1_1s', 'posted count reset before the InitBM (1500)'), + 'reinit-at-any-reclaim': ('same_client_render_after_60ms', 'posted count reset before the InitBM'), + 'freed-without-walk': ('failed_vm_walk_counts_nothing_freed', 'freed counted with the VM walk failed'), 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), - 'no-same-vm-checks': ('grant_refuses_flights_of_two_vms', 'grant with flights of two VMs'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), - 'chunks-above-4mib': ('growth_backs_before_posting', 'a growth chunk above 4 MiB'), + 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), 'pb-leaf-fw-visible': ('pb_leaf_is_gpu_only', 'PB leaf not GPU-only'), } assert set(INTENDED) == set(MUTATIONS) @@ -290,6 +402,26 @@ def consts(text, names): pub fn last(&self) -> Option<&T> { self.0.last() } + pub fn truncate(&mut self, n: usize) { + self.0.truncate(n) + } + pub fn reserve(&mut self, n: usize, _: Flags) -> Result { + self.0.reserve(n); + Ok(()) + } + pub fn iter(&self) -> std::slice::Iter<'_, T> { + self.0.iter() + } +} +/// A kernel mutex, single-threaded here. +pub struct Mutex(pub RefCell); +impl Mutex { + pub fn new(value: T) -> Self { + Mutex(RefCell::new(value)) + } + pub fn lock(&self) -> std::cell::RefMut<'_, T> { + self.0.borrow_mut() + } } #[macro_export] macro_rules! static_assert { @@ -309,20 +441,26 @@ def consts(text, names): /// Backing object `id` of this many bytes allocated, or freed (its last reference gone). Alloc(u32, usize), Free(u32), - /// A PB alias of object `id` at GVA in the VM with this context, made, held by the VM - /// (appended to its driver mappings), or unmapped. + /// A PB alias of object `id` at GVA in the VM with this context, made or unmapped. Map(u32, u64, usize, u32), - Append(u32, u64), Unmap(u32, u64), } thread_local! { pub static LOG: RefCell> = RefCell::new(Vec::new()); pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); - /// Failures to inject: the next this many allocations, alias maps, appends fail. + pub static PARAM_BLOCKS: Cell = Cell::new(PB_BLOCKS_DEFAULT); + /// The reclaim parameters (g15_pb_reclaim, g15_pb_idle_ms, g15_pb_reclaim_blocks). + pub static PARAM_RECLAIM: Cell = Cell::new(PB_RECLAIM_DEFAULT); + pub static PARAM_IDLE_MS: Cell = Cell::new(PB_IDLE_MS_DEFAULT); + pub static PARAM_RECLAIM_BLOCKS: Cell = Cell::new(PB_RECLAIM_BLOCKS_DEFAULT); + /// The monotonic clock (engine::now_ns()). + pub static NOW: Cell = Cell::new(1_000_000_000); + /// The VM registry walk (vms.open()) fails this many times. + pub static FAIL_OPEN: Cell = Cell::new(0); + /// Failures to inject: the next this many allocations or alias maps fail. pub static FAIL_ALLOC: Cell = Cell::new(0); pub static FAIL_MAP: Cell = Cell::new(0); pub static MAP_OK_BEFORE_FAIL: Cell = Cell::new(0); - pub static FAIL_APPEND: Cell = Cell::new(0); pub static NEXT_ID: Cell = Cell::new(1); } /// Takes one injected failure of `which`. @@ -367,6 +505,22 @@ def consts(text, names): } } pub static g15_pb_max_blocks: Param = Param; + pub struct Blocks; + impl Blocks { + pub fn value(&self) -> &'static u32 { + Box::leak(Box::new(crate::PARAM_BLOCKS.with(|p| p.get()))) + } + } + pub static g15_pb_blocks: Blocks = Blocks; + pub struct Cellp(pub &'static std::thread::LocalKey>); + impl Cellp { + pub fn value(&self) -> &'static u32 { + Box::leak(Box::new(self.0.with(|p| p.get()))) + } + } + pub static g15_pb_reclaim: Cellp = Cellp(&crate::PARAM_RECLAIM); + pub static g15_pb_idle_ms: Cellp = Cellp(&crate::PARAM_IDLE_MS); + pub static g15_pb_reclaim_blocks: Cellp = Cellp(&crate::PARAM_RECLAIM_BLOCKS); } pub mod driver { @@ -389,7 +543,7 @@ def consts(text, names): } } -/// The shared layer's VM: kernel aliases at fixed addresses, held by the VM once appended. +/// The shared layer's VM: kernel aliases at fixed addresses. pub mod mmu { use crate::prelude::*; pub struct Prot(pub u64); @@ -412,11 +566,10 @@ def consts(text, names): } pub struct Vm { pub slot: u32, - pub held: std::cell::RefCell>, } impl Vm { pub fn new(slot: u32) -> Self { - Vm { slot, held: Default::default() } + Vm { slot } } pub fn map_at( &self, @@ -437,24 +590,9 @@ def consts(text, names): if !skip && crate::fail(&crate::FAIL_MAP) { return Err(ENOMEM); } - // Leaves already in use: EBUSY (prepare_map). - let end = addr + size as u64; - if self.held.borrow().iter().any(|m| m.gva < end && addr < m.gva + m.size as u64) { - return Err(EBUSY); - } crate::ev(crate::Ev::Map(self.slot, addr, size, gem.id)); Ok(KernelMapping { slot: self.slot, gva: addr, size, prot: prot.0, gem: gem.clone() }) } - pub fn append_driver_mappings(&self, mappings: KVec) -> Result { - if crate::fail(&crate::FAIL_APPEND) { - return Err(ENOMEM); // the mappings drop here - } - for m in mappings.0 { - crate::ev(crate::Ev::Append(self.slot, m.gva)); - self.held.borrow_mut().push(m); - } - Ok(()) - } } } ''' @@ -477,14 +615,26 @@ def consts(text, names): pub struct Init { pub control: Option>, } + /// vm.rs's chunk mapping, its fields as the driver names them. + pub struct HeapMapping { + pub gem: ARef, + pub _mapping: mmu::KernelMapping, + } + pub struct HeapMappings { + pub list: Mutex>, + } pub struct VmState { pub slot: u32, pub vm: mmu::Vm, - pub pb_chunks: AtomicUsize, + pub heap: Arc, } impl VmState { pub fn new(slot: u32) -> Arc { - Arc::new(VmState { slot, vm: mmu::Vm::new(slot), pb_chunks: AtomicUsize::new(0) }) + Arc::new(VmState { + slot, + vm: mmu::Vm::new(slot), + heap: Arc::new(HeapMappings { list: Mutex::new(KVec(Vec::new())) }), + }) } } pub struct Job { @@ -495,11 +645,36 @@ def consts(text, names): } pub struct Engine { pub flights: Vec, + /// Jobs accepted and not fully published. + pub pending: Vec, + pub frozen: bool, + } + pub struct Recovery { + pub reinit: bool, + pub armed: Option<(usize, u64)>, + pub failed: Option, + } + impl Recovery { + pub fn failed(&self) -> Option { + self.failed + } + } + /// vm.rs's Registry: the VMs whose client handle may still be open. + pub struct Registry { + pub vms: Vec>, + } + impl Registry { + pub fn open(&self) -> Result>> { + if crate::fail(&crate::FAIL_OPEN) { + return Err(ENOMEM); + } + Ok(KVec(self.vms.clone())) + } } pub struct Receipts { pub renders: u32, } - /// A firmware object: the block list, block_ctl or the heap. + /// A firmware object: the block list or block_ctl. pub struct FwObject { pub gva: Option, pub size: usize, @@ -532,6 +707,10 @@ def consts(text, names): pub grown: u64, pub refused: u64, pub bad: u64, + pub waits: u64, + pub refills: u64, + pub reclaims: u64, + pub freed: u64, } pub struct Live { pub block_ctl: FwObject, @@ -544,6 +723,8 @@ def consts(text, names): pub live: Live, pub drm: ARef, pub chunks: KVec, + pub idle: growth::IdleClock, + pub reclaim_pending: bool, } impl Pools { pub fn chunk_count(&self) -> usize { @@ -552,6 +733,10 @@ def consts(text, names): pub fn chunk_capacity(&self) -> usize { self.chunks.capacity() } + /// The backing objects' ids, in heap order. + pub fn chunk_ids(&self) -> Vec { + self.chunks.iter().map(|c| c.backing.gem.id).collect() + } } pub struct Queue(pub std::collections::VecDeque<[u32; 14]>); impl Queue { @@ -565,6 +750,11 @@ def consts(text, names): assert_eq!(i, 0); self.0.pop_front().ok_or(()) } + pub fn insert_within_capacity(&mut self, i: usize, q: [u32; 14]) -> core::result::Result<(), [u32; 14]> { + assert_eq!(i, 0); + self.0.push_front(q); + Ok(()) + } } pub struct Events { pub grow: Queue, @@ -597,6 +787,8 @@ def consts(text, names): pub pools: Pools, pub init: Init, pub engine: Engine, + pub recovery: Recovery, + pub vms: Registry, pub renders: Vec, pub fw: FwModel, } @@ -608,11 +800,12 @@ def consts(text, names): let vm = VmState::new(5); let mut f = Self::without_flights(); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + f.vms.vms.push(vm.clone()); f.map_pb(&vm).unwrap(); crate::take(); f } - /// Pools::new's PB backing: the initial chunk, room for every chunk the ledger needs. + /// Pools::new's PB backing: the initial chunks and the reserve, room for the heap. pub fn without_flights() -> Self { let drm = ARef::new(AsahiDevice); let (ledger, chunks) = pools_new_backing(&drm).unwrap(); @@ -621,6 +814,8 @@ def consts(text, names): pools: Pools { ledger, exhausted_logged: false, + idle: growth::IdleClock::default(), + reclaim_pending: false, stats: PbStats::default(), drm, chunks, @@ -634,7 +829,9 @@ def consts(text, names): }, }, init: Init { control: Some(std::sync::Arc::new(Ctl { epoch: 0x1_0000_0003 })) }, - engine: Engine { flights: Vec::new() }, + engine: Engine { flights: Vec::new(), pending: Vec::new(), frozen: false }, + recovery: Recovery { reinit: false, armed: None, failed: None }, + vms: Registry { vms: Vec::new() }, renders: vec![1], fw: FwModel { ctx: 5, @@ -691,10 +888,10 @@ def consts(text, names): use crate::g15::firmware::{Firmware, VmState, Flight, Job, HEAP}; use crate::g15::hw::{client, pte}; use crate::g15::fw::buffer::{BLOCK_BYTES, MAX_BLOCKS}; - use crate::g15::pools::growth::{max_chunks, CHUNK_BLOCKS}; + use crate::g15::pools::growth::{CHUNK_BLOCKS, MAX_CHUNKS}; + use crate::g15::pools::{chunk_gva, CHUNK_BYTES}; use crate::g15::pgtable::Prot; - use crate::{EPROTO, FAIL_ALLOC, FAIL_APPEND, FAIL_MAP}; - use core::sync::atomic::Ordering; + use crate::{EPROTO, FAIL_ALLOC, FAIL_MAP}; use std::sync::Arc; use crate::g15::fw::buffer::{self, grow_reply, BLOCK_LIST_RECORDS, GROW_STEP}; use crate::g15::fw::channels::devctl; @@ -775,7 +972,7 @@ def consts(text, names): fn levels_only(evs: Vec) -> Vec { evs.into_iter() - .filter(|e| !is_backing(e)) + .filter(|e| !is_backing(e) && !matches!(e, Ev::Log(_, m) if m.starts_with("PB heap:"))) .map(|e| match e { Ev::Log(l, _) => Ev::Log(l, String::new()), e => e, @@ -785,6 +982,7 @@ def consts(text, names): /// Growth to the maximum with the firmware asking after every reply, one pass at a time. fn run_to_exhaustion(param: u32) { + defaults(); PARAM.with(|p| p.set(param)); let mut f = Firmware::new(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); @@ -848,9 +1046,11 @@ def consts(text, names): #[test] fn driver_values() { - let l = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + defaults(); + let l = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); assert!(tuning::PB_GROW); - assert_eq!((l.initial, l.max, l.posted), (1024, 4095, 1024)); + assert_eq!(crate::PB_BLOCKS_DEFAULT, 64); + assert_eq!((l.initial, l.max, l.posted), (64, 4095, 64)); assert_eq!(PARAM_DEFAULT, 3072); PARAM.with(|p| p.set(PARAM_DEFAULT)); assert_eq!(params::pb_max_blocks(), 3072); @@ -859,12 +1059,21 @@ def consts(text, names): #[test] fn soft_max_bounds() { + defaults(); let f = Firmware::new(); for param in [0, 1, 63, 64, 65, 1023, 1024, 1025, 3071, 3072, 3073, 4094, 4095, 4096, 8192, u32::MAX] { PARAM.with(|p| p.set(param)); assert_eq!(params::pb_max_blocks(), clamp(param, 64, tuning::PB_MAX_BLOCKS), "pb_max_blocks, param {param}"); - // Below the initial 1024 the firmware still gets 1024. - assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}"); + // Below the initial 64 the firmware still gets 64. + assert_eq!(f.pools.soft_max(), clamp(param, 64, CAP), "soft_max, param {param}"); + } + // With a larger initial posting the firmware gets at least that. + crate::PARAM_BLOCKS.with(|b| b.set(1024)); + let f = Firmware::new(); + assert_eq!(f.pools.ledger.initial, 1024); + for param in [0, 64, 1023, 1024, 3072, 4095, u32::MAX] { + PARAM.with(|p| p.set(param)); + assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}, 1024 initial"); } } @@ -875,7 +1084,7 @@ def consts(text, names): #[test] fn soft_max_boundaries() { - // 3074 is reached exactly (1024 + 205 * 10): that answer is not past it. + // 3074 is reached exactly (64 + 301 * 10): that answer is not past it. for param in [0, 1024, 3072, 3074, 3075, 4094, 4095, u32::MAX] { run_to_exhaustion(param); } @@ -888,6 +1097,7 @@ def consts(text, names): f.queue(); f.queue(); take(); + let mapped = f.pools.ledger.backed as u64; f.pass(); let log = take(); let replies: Vec = log @@ -931,7 +1141,7 @@ def consts(text, names): assert_eq!(f.pools.stats.requests, 2, "requests taken in the pass"); assert_eq!(f.pools.stats.bad, 1); assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -5")))); - assert!(log.contains(&Ev::Producer(1024 + GROW_STEP))); + assert!(log.contains(&Ev::Producer(64 + GROW_STEP))); assert!(f.events.grow.is_empty()); } @@ -945,7 +1155,7 @@ def consts(text, names): take(); f.pass(); let log = take(); - assert_eq!(f.pools.ledger.posted, 1024 + 3 * GROW_STEP); + assert_eq!(f.pools.ledger.posted, 64 + 3 * GROW_STEP); assert_eq!((f.pools.stats.grown, f.pools.stats.bad), (3 * GROW_STEP as u64, 0)); // Each failed reply is logged at info level. let info = log.iter().filter(|e| matches!(e, Ev::Log("info", m) if m.contains("DEVCTL 7: -5"))).count(); @@ -978,7 +1188,9 @@ def consts(text, names): assert!(log.iter().all(|e| matches!(e, Ev::Log("warn", _))), "{log:?}"); assert_eq!(log.len(), 4); assert_eq!(f.pools.stats.bad, 4); - assert_eq!(f.pools.ledger, Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS)); + let mut fresh = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); + fresh.backed = Ledger::chunks_for(fresh.initial) * CHUNK_BLOCKS; + assert_eq!(f.pools.ledger, fresh); // No render published: not live. f.renders[0] = 0; f.events.grow.0.push_back(good); @@ -996,54 +1208,43 @@ def consts(text, names): } fn is_backing(e: &Ev) -> bool { - matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Append(..) | Ev::Unmap(..)) - } - - /// A VM's PB aliases: its count, and the aliases it holds, from block 0 in order. - fn assert_cursor(vm: &VmState, chunks: usize) { - let held = vm.vm.held.borrow(); - assert_eq!(vm.pb_chunks.load(Ordering::Relaxed), held.len(), "count differs from held aliases"); - assert!(held.len() <= chunks); - let mut gva = client::PB_HEAP; - for m in held.iter() { - assert_eq!(m.gva, gva, "aliases not contiguous from the heap's start"); - assert_eq!(m.prot, pte::LEAF_CACHED); - gva += m.size as u64; + matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Unmap(..)) + } + + /// A VM's PB aliases: exactly the first `chunks` chunks, in order, each the whole chunk at + /// its GVA with the PB leaf, each of the chunk's object. + fn assert_cursor(f: &Firmware, vm: &VmState, chunks: usize) { + let heap = vm.heap.list.lock(); + assert_eq!(heap.len(), chunks, "VM aliases differ from the chunks"); + let ids = f.pools.chunk_ids(); + for (i, m) in heap.iter().enumerate() { + assert_eq!(m._mapping.gva, chunk_gva(i), "aliases not contiguous from the heap's start"); + assert_eq!(m._mapping.size, CHUNK_BYTES); + assert_eq!(m._mapping.prot, pte::LEAF_CACHED); + assert_eq!(m.gem.id, ids[i], "alias of another object"); } } /// In the order of `log`: no block record and no producer past the blocks the VM with - /// context `slot` holds aliases for at that point. + /// context `slot` has mapped at that point (`start` blocks before the log). fn assert_posted_backed(log: &[Ev], slot: u32, start: u64) { let mut mapped = start; for e in log { match e { - Ev::Append(s, gva) if *s == slot => { - assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "append out of order"); + Ev::Map(s, gva, size, _) if *s == slot => { + assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "map out of order"); + mapped += *size as u64 / BLOCK_BYTES; } - Ev::Map(s, gva, size, _) if *s == slot && *gva == client::PB_HEAP + mapped * BLOCK_BYTES => { - // Counted at its append. - let _ = size; + Ev::Unmap(s, gva) if *s == slot => { + mapped = mapped.min((gva - client::PB_HEAP) / BLOCK_BYTES); } - _ => {} - } - if let Ev::Append(s, gva) = e { - if *s == slot { - let size = log - .iter() - .find_map(|m| match m { - Ev::Map(ms, mg, size, _) if ms == s && mg == gva => Some(*size as u64), - _ => None, - }) - .unwrap(); - mapped += size / BLOCK_BYTES; + Ev::Record(off, _) => { + assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); } - } - if let Ev::Record(off, _) = e { - assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); - } - if let Ev::Producer(p) = e { - assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + Ev::Producer(p) => { + assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + } + _ => {} } } } @@ -1069,17 +1270,20 @@ def consts(text, names): let mut f = Firmware::without_flights(); let log = take(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); - assert_eq!(log, vec![Ev::Alloc(1, initial as usize * BLOCK_BYTES as usize)], "startup allocation"); - assert_eq!((f.pools.backed(), f.pools.chunk_count()), (initial, 1)); - assert_eq!(f.pools.chunk_capacity(), max_chunks(initial, max)); - assert_eq!(max_chunks(initial, max), 97); + let chunks = Ledger::chunks_for(initial) as usize; + assert_eq!(chunks, 2); + let want: Vec = (1..=chunks as u32).map(|id| Ev::Alloc(id, CHUNK_BYTES)).collect(); + assert_eq!(log, want, "startup allocation"); + assert_eq!((f.pools.ledger.backed, f.pools.chunk_count()), (chunks as u32 * CHUNK_BLOCKS, chunks)); + assert_eq!(f.pools.chunk_capacity(), MAX_CHUNKS as usize); + assert!(max <= MAX_CHUNKS * CHUNK_BLOCKS); // The first prefix of a VM maps the backing, nothing else. let vm = VmState::new(7); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); f.map_pb(&vm).unwrap(); - let size = initial as usize * BLOCK_BYTES as usize; - assert_eq!(take(), vec![Ev::Map(7, client::PB_HEAP, size, 1), Ev::Append(7, client::PB_HEAP)]); - assert_cursor(&vm, 1); + let want: Vec = (0..chunks).map(|i| Ev::Map(7, chunk_gva(i), CHUNK_BYTES, i as u32 + 1)).collect(); + assert_eq!(take(), want); + assert_cursor(&f, &vm, chunks); // Mapping again maps nothing. f.map_pb(&vm).unwrap(); assert!(take().is_empty()); @@ -1093,8 +1297,10 @@ def consts(text, names): #[test] fn growth_backs_before_posting() { - PARAM.with(|p| p.set(PARAM_DEFAULT)); + defaults(); let mut f = Firmware::new(); + let start = f.pools.ledger.backed as u64; + let first_chunks = f.pools.chunk_count(); f.fw.ask_after_reply = true; f.queue(); take(); @@ -1105,77 +1311,179 @@ def consts(text, names): assert!(passes <= 1000, "growth did not end"); } let log = take(); - let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); - assert_posted_backed(&log, 5, initial as u64); - // Chunks of at most 32 blocks, allocated only as needed: up to the maximum, no further. + assert_posted_backed(&log, 5, start); + // 16 MiB chunks, backed as the reserve needs them: up to the whole heap, no further. let allocs: Vec = log .iter() .filter_map(|e| if let Ev::Alloc(_, b) = e { Some(*b) } else { None }) .collect(); - assert_eq!(allocs.len() + 1, max_chunks(initial, max)); - assert!(allocs.iter().all(|b| *b <= 4 << 20), "a growth chunk above 4 MiB"); - assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 4 << 20); - assert_eq!(f.pools.backed(), max); + assert!(allocs.iter().all(|b| *b <= 16 << 20), "a growth chunk above 16 MiB"); + assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 16 << 20); + assert_eq!(allocs.len() + first_chunks, MAX_CHUNKS as usize); + assert_eq!(f.pools.ledger.backed, MAX_BLOCKS); + assert_eq!(f.pools.stats.waits, 0); + assert_eq!(f.pools.stats.refills, allocs.len() as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); - assert_cursor(&f.engine.flights[0].job.vm, f.pools.chunk_count()); + assert_cursor(&f, &f.engine.flights[0].job.vm, f.pools.chunk_count()); } - /// A failure while backing posts nothing and answers nothing; the request counter and the - /// VM's count stay; the next request is answered. - fn failure_posts_nothing(which: &'static std::thread::LocalKey>) -> Vec { + #[test] + fn allocation_follows_the_replies() { let mut f = Firmware::new(); - which.with(|c| c.set(1)); + // Below the reserve line: the pass's answers come first, then the chunk. + f.pools.ledger.posted = f.pools.ledger.backed - growth_reserve() + 1; f.queue(); - f.pass(); - let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_) | Ev::Sync)), 0, - "posted after a backing failure: {log:?}"); - assert_eq!((f.pools.ledger.posted, f.pools.ledger.next_request), (1024, 0), "ledger after a backing failure"); - assert_eq!((f.pools.stats.bad, f.pools.chunk_count()), (1, 1), "chunks after a backing failure"); - assert_cursor(&f.engine.flights[0].job.vm, 1); - assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -12")))); f.queue(); + take(); f.pass(); - let retry = take(); - assert!(retry.contains(&Ev::Producer(1034))); - assert_eq!(f.pools.chunk_count(), 2); - assert_cursor(&f.engine.flights[0].job.vm, 2); - log + let log = take(); + let last_reply = log.iter().rposition(|e| matches!(e, Ev::Reply(_))).expect("replies"); + let first_alloc = log.iter().position(|e| matches!(e, Ev::Alloc(..))).expect("a refill"); + assert!(first_alloc > last_reply, "allocation before a reply: {log:?}"); + assert_eq!(count(&log, |e| matches!(e, Ev::Reply(_))), 2); + assert_eq!(f.pools.stats.refills, 1); + } + + fn growth_reserve() -> u32 { + crate::g15::pools::growth::RESERVE_BLOCKS } #[test] - fn allocation_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_ALLOC); - assert_eq!(count(&log, is_backing), 0); + fn dry_reserve_waits_and_keeps_order() { + let mut f = Firmware::new(); + let start = f.pools.ledger.backed as u64; + FAIL_ALLOC.with(|c| c.set(1000)); + let mut log = Vec::new(); + let mut waited = None; + for _ in 0..40 { + f.queue(); + f.pass(); + let pass = take(); + let replied = pass.iter().any(|e| matches!(e, Ev::Reply(_))); + log.extend(pass); + if !replied { + waited = Some(f.fw.next_counter - 1); + break; + } + } + assert_posted_backed(&log, 5, start); + let waited = waited.expect("no request waited"); + assert_eq!(f.pools.ledger.posted, f.pools.ledger.backed); + let queued: Vec = f.events.grow.0.iter().map(|q| q[buffer::grow_request::COUNTER]).collect(); + assert_eq!(queued, vec![waited], "waiting request lost"); + assert_eq!(f.pools.ledger.next_request, waited); + assert!(f.pools.stats.waits >= 1); + // The allocator recovers: this pass still waits, then backs a chunk; the next answers. + FAIL_ALLOC.with(|c| c.set(0)); + f.pass(); + let log = take(); + assert!(!log.iter().any(|e| matches!(e, Ev::Reply(_)))); + assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 1); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 1); + let mapped = f.pools.ledger.backed as u64; + f.pass(); + let log = take(); + let replies: Vec = log + .iter() + .filter_map(|e| if let Ev::Reply(r) = e { Some(r[grow_reply::COUNTER]) } else { None }) + .collect(); + assert_eq!(replies, vec![waited]); + assert!(f.events.grow.is_empty()); + assert_posted_backed(&log, 5, mapped); } #[test] - fn map_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_MAP); + fn refill_map_failure_drops_the_chunk() { + let mut f = Firmware::new(); + let chunks = f.pools.chunk_count(); + let backed = f.pools.ledger.backed; + f.pools.ledger.posted = backed - growth_reserve() + 1; + FAIL_MAP.with(|c| c.set(1)); + take(); + f.pass(); + let log = take(); + assert_eq!(f.pools.chunk_count(), chunks, "chunk kept after a failed map"); let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); - assert!(matches!(backing[..], [Ev::Alloc(2, _), Ev::Free(2)]), "{backing:?}"); + assert!(matches!(backing[..], [Ev::Alloc(id, _), Ev::Free(free)] if id == free), "{backing:?}"); + assert_eq!(f.pools.ledger.backed, backed); + assert!(log.iter().any(|e| matches!(e, Ev::Log("warn", m) if m.contains("backing chunk")))); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks); + // The next pass backs it. + f.pass(); + assert_eq!(f.pools.chunk_count(), chunks + 1); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks + 1); } + /// A pending re-initialization with nothing published (a reclaim, or a recovery that + /// failed the flights): the posted count is about to drop, so nothing is backed. #[test] - fn append_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_APPEND); - let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); + fn pending_reinit_without_flights_backs_nothing() { + let mut f = Firmware::without_flights(); + f.pools.ledger.posted = f.pools.ledger.backed - growth_reserve() + 1; + f.recovery.reinit = true; + take(); + f.pass(); + assert_eq!(count(&take(), is_backing), 0, "backed during a pending re-initialization"); + f.recovery.reinit = false; + f.pass(); + assert_eq!(f.pools.stats.refills, 1); + } + + /// A pending re-initialization with only compute published (no render can ask for + /// growth): nothing is backed. + #[test] + fn pending_reinit_with_compute_only_backs_nothing() { + let mut f = Firmware::new(); + f.renders[0] = 0; + f.pools.ledger.posted = f.pools.ledger.backed; + f.recovery.reinit = true; + take(); + for _ in 0..4 { + f.pass(); + } + assert_eq!(count(&take(), is_backing), 0, "backed for a compute-only flight"); + assert_eq!(f.pools.stats.refills, 0); + } + + /// A render published before the re-initialization was set (the flights behind a fragment + /// kill) still asks for growth: with its reserve used up, the refill must still back + /// chunks, or the request waits until the deadline. + #[test] + fn pending_reinit_with_a_live_render_refills() { + let mut f = Firmware::new(); // one live render + let start = f.pools.ledger.backed as u64; + f.pools.ledger.posted = f.pools.ledger.backed; + f.recovery.reinit = true; + f.queue(); + take(); + for _ in 0..4 { + f.pass(); + } + let s = &f.pools.stats; assert!( - matches!(backing[..], [Ev::Alloc(2, _), Ev::Map(5, _, _, 2), Ev::Unmap(5, _), Ev::Free(2)]), - "{backing:?}" + f.events.grow.is_empty(), + "growth request stranded: posted={} backed={} waits={} refills={}", + f.pools.ledger.posted, + f.pools.ledger.backed, + s.waits, + s.refills ); + // The request waits one pass for the chunk; the grant's pass tops the reserve up. + assert_eq!((s.waits, s.refills, s.requests), (1, 2, 1)); + assert_posted_backed(&take(), 5, start); } #[test] fn transport_failure_keeps_everything() { let mut f = Firmware::new(); + let chunks = f.pools.chunk_count(); f.fw.reply_fails = true; f.queue(); f.pass(); let log = take(); - assert!(log.contains(&Ev::Producer(1034))); - assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (1034, 2)); - assert_cursor(&f.engine.flights[0].job.vm, 2); + assert!(log.contains(&Ev::Producer(74))); + assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (74, chunks)); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); } @@ -1183,9 +1491,10 @@ def consts(text, names): fn other_vms_catch_up_before_they_run() { let mut f = Firmware::new(); let a = f.engine.flights[0].job.vm.clone(); - grow_to(&mut f, 1100); + grow_to(&mut f, 200); take(); - assert_eq!(f.pools.chunk_count(), 4); + let chunks = f.pools.chunk_count(); + assert_eq!(chunks, 3); // A's work retires; a new VM runs next: it maps every chunk first. f.engine.flights.clear(); let b = VmState::new(6); @@ -1193,84 +1502,53 @@ def consts(text, names): f.fw.ctx = 6; f.map_pb(&b).unwrap(); let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Append(6, _))), 4); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(6, ..))), chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 0); - assert_cursor(&b, 4); + assert_cursor(&f, &b, chunks); // B grows the backing; A, behind, catches up when it runs again. - grow_to(&mut f, 1200); + grow_to(&mut f, 300); let log = take(); - assert_posted_backed(&log, 6, 1120); + assert_posted_backed(&log, 6, chunks as u64 * CHUNK_BLOCKS as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 0); + assert_eq!(f.pools.chunk_count(), chunks + 1); f.engine.flights.clear(); f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); f.map_pb(&a).unwrap(); - assert_cursor(&a, f.pools.chunk_count()); - assert_eq!(a.pb_chunks.load(Ordering::Relaxed), f.pools.chunk_count()); - } - - #[test] - fn growth_catches_the_running_vm_up_first() { - let mut f = Firmware::new(); - let a = f.engine.flights[0].job.vm.clone(); - // B grows the backing while A is idle. - f.engine.flights.clear(); - let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.map_pb(&b).unwrap(); - f.fw.ctx = 6; - grow_to(&mut f, 1100); - // A runs again; its growth request maps what it lacks before backing more. - f.engine.flights.clear(); - f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); - f.fw.ctx = 5; - take(); - grow_to(&mut f, 1130); - let log = take(); - assert_eq!(a.vm.held.borrow().len(), f.pools.chunk_count(), "running VM not caught up"); - assert_cursor(&a, f.pools.chunk_count()); - assert_posted_backed(&log, 5, 1024); + assert_cursor(&f, &a, f.pools.chunk_count()); } #[test] - fn partial_catch_up_counts_only_held_aliases() { + fn partial_catch_up_keeps_the_aliases_made() { let mut f = Firmware::new(); - grow_to(&mut f, 1100); + grow_to(&mut f, 200); + let chunks = f.pools.chunk_count(); f.engine.flights.clear(); let b = VmState::new(6); f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); take(); // The third alias cannot be made: two held. - FAIL_MAP.with(|c| c.set(0)); crate::MAP_OK_BEFORE_FAIL.with(|c| c.set(2)); FAIL_MAP.with(|c| c.set(1)); assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_cursor(&b, 4); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2); - // The third alias is made but cannot be held: dropped, still two. - FAIL_APPEND.with(|c| c.set(1)); - assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2, "count after a failed append"); - assert!(take().iter().any(|e| matches!(e, Ev::Unmap(6, _)))); - assert_cursor(&b, 4); + assert_cursor(&f, &b, 2); f.map_pb(&b).unwrap(); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 4); - assert_cursor(&b, 4); + assert_cursor(&f, &b, chunks); } /// The ledger reset of Pools::init(), not a full recovery. #[test] fn ledger_reset_reuses_the_backing() { let mut f = Firmware::new(); - grow_to(&mut f, 1300); + grow_to(&mut f, 300); let chunks = f.pools.chunk_count(); take(); f.reset_ledger(); - assert_eq!(f.pools.ledger.posted, 1024); - grow_to(&mut f, 1300); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 300); let log = take(); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..) | Ev::Map(..))), 0); assert_eq!(f.pools.chunk_count(), chunks); - assert_posted_backed(&log, 5, f.pools.backed() as u64); + assert_posted_backed(&log, 5, f.pools.ledger.backed as u64); } #[test] @@ -1285,36 +1563,421 @@ def consts(text, names): assert_eq!(f.pools.stats.bad, 1); } + /// Every PB parameter at its default. libtest gives each test its own thread, but a test + /// that sets one starts from the defaults all the same. + fn defaults() { + PARAM.with(|p| p.set(PARAM_DEFAULT)); + crate::PARAM_BLOCKS.with(|b| b.set(crate::PB_BLOCKS_DEFAULT)); + crate::PARAM_RECLAIM.with(|r| r.set(crate::PB_RECLAIM_DEFAULT)); + crate::PARAM_IDLE_MS.with(|r| r.set(crate::PB_IDLE_MS_DEFAULT)); + crate::PARAM_RECLAIM_BLOCKS.with(|r| r.set(crate::PB_RECLAIM_BLOCKS_DEFAULT)); + } + + /// A prefix of `vm` published as engine.rs does it: publish_prefix_into() first catches the + /// VM up (map_pb(), before the flight is added), and a render prefix with a + /// re-initialization pending does the InitBM (Pools::init()'s ledger reset) and clears + /// `reinit` at its publication. `render`: the prefix has renders. + fn publish(f: &mut Firmware, vm: &Arc, render: bool) { + f.map_pb(vm).unwrap(); + if render && f.recovery.reinit { + f.reset_ledger(); + f.recovery.reinit = false; + } + f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + let i = f.engine.flights.len() - 1; + if f.renders.len() <= i { + f.renders.resize(i + 1, 0); + } + f.renders[i] = render as u32; + f.fw.ctx = vm.slot; + f.note_idle(); + } + + /// What a reclaim leaves behind in a VM that was not walked (a closed VM no longer in the + /// registry, whose accepted jobs may still run): its aliases of freed chunks go, and chunks + /// backed again at the same GVA are mapped from their new objects at its next publication. + /// The reclaims are the actual idle reclaim; the posted count stays until the InitBM. #[test] - fn catch_up_refuses_flights_of_another_vm() { + fn freed_chunks_are_mapped_again() { + defaults(); let mut f = Firmware::new(); let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(chunks > 2); + // A's work retires and A leaves the registry; the idle reclaim frees the chunks above + // the kept two without walking A. + f.engine.flights.clear(); + f.vms.vms.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + take(); + // B runs: its first render re-initializes; it grows the backing again, new objects at + // the old GVAs. let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - assert_eq!(f.map_pb(&b), Err(EPROTO), "catch-up with a flight of another VM"); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 0); - assert_eq!(count(&take(), is_backing), 0); - // A count past the chunks. - f.engine.flights.pop(); - a.pb_chunks.store(f.pools.chunk_count() + 1, Ordering::Relaxed); - assert_eq!(f.map_pb(&a), Err(EPROTO), "catch-up with a count past the chunks"); + f.vms.vms.push(b.clone()); + publish(&mut f, &b, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 1100); + assert_eq!(f.pools.chunk_count(), chunks); + // A runs again: its stale aliases are replaced by the new objects'. + f.engine.flights.clear(); + take(); + publish(&mut f, &a, true); + let log = take(); + assert_cursor(&f, &a, chunks); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), chunks - 2); + // Another idle reclaim walks B, not A: with fewer chunks than A maps, A's aliases past + // the backing go at its next catch-up. + f.engine.flights.clear(); + let t1 = now(); + check_at(&mut f, t1); + check_at(&mut f, t1 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + assert_cursor(&f, &b, 2); + f.map_pb(&a).unwrap(); + assert_cursor(&f, &a, 2); } + /// A reclaim whose walk of the VM registry fails: the chunks leave the backing, but a VM's + /// aliases still hold their pages, so nothing is counted freed; the VM drops its stale + /// aliases at its next publication, and only then are the pages released. #[test] - fn grant_refuses_flights_of_two_vms() { + fn failed_vm_walk_counts_nothing_freed() { + defaults(); let mut f = Firmware::new(); - let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.renders.push(1); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1500); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + take(); + crate::FAIL_OPEN.with(|c| c.set(1)); + check_at(&mut f, t0 + 1_000_000_000); + let log = take(); + assert_eq!(f.pools.chunk_count(), 2); + assert_eq!(f.pools.stats.freed, 0, "freed counted with the VM walk failed"); + assert!(log.iter().any(|e| matches!(e, Ev::Log("warn", m) if m.contains("could not walk the VMs")))); + assert!(log.iter().any(|e| matches!(e, Ev::Log("info", m) if m.contains(" 0 chunks freed")))); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), 0, "pages released while aliases hold them"); + assert_eq!(a.heap.list.lock().len(), chunks); + // A's next publication drops the stale aliases: the pages go. + publish(&mut f, &a, true); + let log = take(); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + assert_cursor(&f, &a, 2); + } + + /// One client idles 60 ms after a heavy scene (more than half the maximum posted) and + /// renders again: the client-boundary re-initialization was set at 50 ms with the backing + /// kept, the posted count stays until the render's InitBM, which posts the initial blocks + /// again, and the scene grows again from the kept backing with nothing allocated or mapped. + #[test] + fn same_client_render_after_60ms() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 60_000_000); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!( + (f.pools.ledger.posted, f.pools.chunk_count()), + (posted, chunks), + "posted count reset before the InitBM" + ); + take(); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + assert!(!f.recovery.reinit && !f.pools.reclaim_pending); + grow_to(&mut f, 2100); + let log = take(); + assert_eq!(count(&log, is_backing), 0, "backing changed: {:?}", log.iter().filter(|e| is_backing(e)).collect::>()); + assert_posted_backed(&log, 5, f.pools.ledger.backed as u64); + } + + /// One client idles 1.1 s and renders again, after a scene above the reclaim floor (the + /// idle reclaim at 1 s) or above half the maximum (the client boundary at 50 ms, then the + /// free at 1 s): the backing above the kept chunks was freed and unmapped from the client's + /// VM, the posted count stays until the render's InitBM, and the scene grows again with + /// every chunk backed and mapped before a record or producer names it. + #[test] + fn same_client_render_after_1_1s() { + for target in [1500, 2100] { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, target); + let posted = f.pools.ledger.posted; + f.engine.flights.clear(); + let t0 = now(); + for dt in [0, 50_000_000, 1_000_000_000, 1_100_000_000] { + check_at(&mut f, t0 + dt); + } + assert_eq!(f.pools.chunk_count(), 2, "not freed ({target})"); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM ({target})"); + assert_cursor(&f, &a, 2); + take(); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, target); + let log = take(); + assert_posted_backed(&log, 5, 2 * CHUNK_BLOCKS as u64); + assert_cursor(&f, &a, f.pools.chunk_count()); + } + } + + /// A compute-only submission between a level-2 reclaim and the next render's InitBM: its + /// publication maps nothing, its passes back and post nothing (the posted count is still + /// the old one, above the kept backing), and a growth request naming its context is left + /// unanswered; once it retires, the render after it re-initializes as usual. + #[test] + fn compute_between_reclaim_and_initbm() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1500); + let posted = f.pools.ledger.posted; + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + let bad = f.pools.stats.bad; + take(); + publish(&mut f, &a, false); f.queue(); - f.pass(); + for _ in 0..4 { + f.pass(); + } + let log = take(); + let touched: Vec<&Ev> = log + .iter() + .filter(|e| is_backing(e) || matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_))) + .collect(); + assert!(touched.is_empty(), "compute-only flight backed or posted: {touched:?}"); + assert!(f.recovery.reinit && f.pools.ledger.posted == posted); + assert_eq!(f.pools.stats.bad, bad + 1); + // The compute retires; the check finds nothing more to do; the next render re-inits. + f.engine.flights.clear(); + check_at(&mut f, t0 + 2_000_000_000); + assert_eq!(count(&take(), is_backing), 0); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 300); + assert_posted_backed(&take(), 5, 2 * CHUNK_BLOCKS as u64); + } + + fn now() -> u64 { + crate::NOW.with(|n| n.get()) + } + + /// The idle clock at `t` and the reclaim check at that time, as a job arrival or an idle + /// poll pass runs it. + fn check_at(f: &mut Firmware, t: u64) { + crate::NOW.with(|n| n.set(t)); + f.note_idle(); + f.reclaim_check(); + } + + /// A heavy client (more than half the maximum posted) exits and nothing renders after it: + /// the 50 ms client-boundary re-initialization keeps the backing, and a check after the + /// long idle frees the backing above the kept chunks. The posted count stays until the next + /// render's InitBM. + #[test] + fn client_boundary_reinit_frees_without_a_render() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(posted > 2047 && chunks > 2); + // The pass that retires the last flight notes the idle. + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + take(); + check_at(&mut f, t0 + 50_000_000); + assert!(f.recovery.reinit && f.pools.chunk_count() == chunks, "client-boundary re-initialization"); + check_at(&mut f, t0 + 999_999_999); + assert_eq!(f.pools.chunk_count(), chunks, "freed before the long idle"); + check_at(&mut f, t0 + 1_000_000_000); + let log = take(); + assert_eq!(f.pools.chunk_count(), 2, "kept backing not freed: {} chunks", f.pools.chunk_count()); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!(f.pools.stats.reclaims, 1); + assert_eq!(f.pools.ledger.backed, 2 * CHUNK_BLOCKS); + assert_eq!(f.pools.stats.freed, (chunks - 2) as u64); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + // Later checks find nothing more to do. + check_at(&mut f, t0 + 100_000_000_000); + assert!(take().is_empty()); + } + + /// With g15_pb_reclaim 1 the client-boundary re-initialization keeps the backing for good. + #[test] + fn client_boundary_reinit_at_level_1_keeps_the_backing() { + defaults(); + crate::PARAM_RECLAIM.with(|r| r.set(1)); + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 50_000_000); + assert!(f.recovery.reinit); + assert_eq!(f.reclaim_delay_ns(), None, "armed at level 1"); + check_at(&mut f, t0 + 5_000_000_000); + assert_eq!(f.pools.chunk_count(), chunks, "freed at level 1"); + assert_eq!(f.pools.stats.freed, 0); + } + + /// Nothing grown since the initialization: an initial posting above half the maximum + /// (g15_pb_blocks 3000), or a floor below it (g15_pb_reclaim_blocks 32): no + /// re-initialization at any idle. + #[test] + fn nothing_grown_reinitializes_nothing() { + for (blocks, floor) in [(64, 32), (3000, 1024)] { + defaults(); + crate::PARAM_BLOCKS.with(|b| b.set(blocks)); + crate::PARAM_RECLAIM_BLOCKS.with(|b| b.set(floor)); + let mut f = Firmware::new(); + assert_eq!(f.pools.ledger.posted, blocks); + f.engine.flights.clear(); + let t0 = now(); + for dt in [0, 50_000_000, 1_000_000_000, 5_000_000_000] { + check_at(&mut f, t0 + dt); + } + assert!( + !f.recovery.reinit && f.pools.stats.reclaims == 0, + "re-initialization with nothing grown ({blocks} blocks, floor {floor})" + ); + } + } + + /// A recovery's re-initialization (not a reclaim's) is left alone: nothing freed. + #[test] + fn recovery_reinit_frees_nothing() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + f.recovery.reinit = true; + let t0 = now(); + check_at(&mut f, t0); + assert_eq!(f.reclaim_delay_ns(), None, "armed for a recovery's re-initialization"); + check_at(&mut f, t0 + 5_000_000_000); + assert_eq!(f.pools.chunk_count(), chunks, "freed for a recovery's re-initialization"); + assert_eq!(f.pools.stats.reclaims, 0); + } + + /// The idle poll work as poll.rs drives it: a pass with nothing published notes the idle + /// and runs the reclaim check; an idle pass is re-armed only for reclaim_delay_ns(), + /// rounded up to whole ms, plus one 4 ms jiffy. Returns the wake times. + fn idle_polls(f: &mut Firmware, limit: usize) -> Vec { + let mut wakes = Vec::new(); + loop { + f.note_idle(); + f.reclaim_check(); + let Some(ns) = f.reclaim_delay_ns() else { + return wakes; + }; + assert!(wakes.len() < limit, "poll woke in a loop: {} wakes, next in {ns} ns", wakes.len()); + let t = now() + ns.div_ceil(1_000_000) * 1_000_000 + 4_000_000; + crate::NOW.with(|n| n.set(t)); + wakes.push(t); + } + } + + /// The same exit with the idle poll work: the poll is armed for the 50 ms + /// re-initialization, then for the rest of the long idle (not the expired 50 ms), where the + /// kept backing is freed; then nothing is armed. + #[test] + fn client_boundary_reinit_arms_the_long_idle() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(posted > 2047 && chunks > 2); + // The pass that retires the last flight notes the idle. + f.engine.flights.clear(); + f.note_idle(); + let t0 = now(); + take(); + let wakes = idle_polls(&mut f, 8); let log = take(); assert_eq!( - count(&log, |e| is_backing(e) || matches!(e, Ev::Reply(_) | Ev::Producer(_) | Ev::Record(..))), - 0, - "grant with flights of two VMs" + f.pools.chunk_count(), + 2, + "kept backing not freed: {} chunks, wakes {:?} ms", + f.pools.chunk_count(), + wakes.iter().map(|t| (t - t0) / 1_000_000).collect::>() ); - assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -71")))); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!(f.pools.stats.reclaims, 1); + assert_eq!(f.pools.ledger.backed, 2 * CHUNK_BLOCKS); + assert_eq!(f.pools.stats.freed, (chunks - 2) as u64); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + // Two wakes: the re-initialization after 50 ms, the free after the long idle. + let ms: Vec = wakes.iter().map(|t| (t - t0) / 1_000_000).collect(); + assert_eq!(ms, vec![54, 1004], "wakes (ms after the idle began)"); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + // Idle much longer, or a job arriving: nothing more is due. + crate::NOW.with(|n| n.set(t0 + 100_000_000_000)); + f.reclaim_check(); + assert_eq!(f.reclaim_delay_ns(), None); + assert!(take().is_empty()); + } + + /// Nothing grown: the idle poll work arms nothing. + #[test] + fn nothing_grown_arms_nothing() { + for (blocks, floor) in [(64, 32), (3000, 1024)] { + defaults(); + crate::PARAM_BLOCKS.with(|b| b.set(blocks)); + crate::PARAM_RECLAIM_BLOCKS.with(|b| b.set(floor)); + let mut f = Firmware::new(); + f.engine.flights.clear(); + let wakes = idle_polls(&mut f, 8); + assert!(wakes.is_empty(), "armed with nothing grown ({blocks} blocks, floor {floor}): {} wakes", wakes.len()); + assert!(!f.recovery.reinit); + } + } + + /// An idle reclaim below half the maximum frees at the long idle; with its re-initialization + /// pending nothing more is armed. + #[test] + fn idle_reclaim_arms_nothing_after() { + let mut f = Firmware::new(); + grow_to(&mut f, 1500); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + f.note_idle(); + let t0 = now(); + let wakes = idle_polls(&mut f, 8); + let ms: Vec = wakes.iter().map(|t| (t - t0) / 1_000_000).collect(); + assert_eq!(ms, vec![1004]); + assert_eq!((f.pools.chunk_count(), f.pools.stats.freed), (2, (chunks - 2) as u64)); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + } + + #[test] + fn catch_up_refuses_flights_of_another_vm() { + let mut f = Firmware::new(); + let b = VmState::new(6); + f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); + assert_eq!(f.map_pb(&b), Err(EPROTO), "catch-up with a flight of another VM"); + assert_eq!(b.heap.list.lock().len(), 0); + assert_eq!(count(&take(), is_backing), 0); } #[test] @@ -1346,7 +2009,7 @@ def consts(text, names): // Only inputs the old constructor took to the whole 4096-block heap changed. assert!(changed > 0 && changed < cases); // The driver's values are below the cap, so its ledger is the old one. - let d = (tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let d = (tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); let (old, new) = (OldLedger::new(d.0, d.1, d.2), Ledger::new(d.0, d.1, d.2)); if d.2 <= CAP { assert_eq!((old.initial, old.max, old.posted), (new.initial, new.max, new.posted)); @@ -1384,7 +2047,7 @@ def read(path, ref=None): raise SystemExit('embedded Ledger::new() differs from ' + OLD_REF) def run(mutation): - paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE) + paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE, VM) src = {path: read(path) for path in paths} if mutation: spec = {**MUTATIONS, **CONTROLS}[mutation] @@ -1394,15 +2057,28 @@ def run(mutation): src[path] = src[path].replace(old, new) pools, hw = src[POOLS], src[HW] bodies = [item(pools, s) for s in (' pub(crate) fn answer_growth(&mut self) -> Result {', - ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', + ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', ' pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result {', - ' fn running_vm(&self, ctx: u32) -> Result> {', - ' fn back_pb(&mut self, end: u32, ctx: u32) -> Result {')] + ' pub(crate) fn refill_backing(&mut self) {', + ' pub(crate) fn note_idle(&mut self) {', + ' pub(crate) fn reclaim_check(&mut self) {', + ' fn free_backing(&mut self) -> usize {', + ' pub(crate) fn reclaim_delay_ns(&self) -> Option {')] soft_max = item(pools, ' pub(crate) fn soft_max(&self) -> u32 {') - backed = item(pools, ' pub(crate) fn backed(&self) -> u32 {') + pools_methods = (item(pools, ' fn map_heap(&self, vm: &VmState) -> Result {') + '\n' + + item(pools, ' fn free_chunks(&mut self, keep: usize) -> usize {') + '\n' + + item(pools, ' fn back_chunk(&mut self) -> Result> {')) chunk = item(pools, 'struct PbChunk {') + '\n' + item(pools, 'impl PbChunk {') + chunk_geometry = (re.search(r'^pub\(crate\) const CHUNK_BYTES: usize = [^;]+;', pools, re.M).group(0) + '\n' + + item(pools, 'pub(crate) const fn chunk_gva(i: usize) -> u64 {')) + vm_src = src[VM] + vm_methods = '\n'.join(item(vm_src, s) for s in ( + ' fn map_one(&self, heap: &mut KVec, i: usize, gem: ARef) -> Result {', + ' pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result {', + ' pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result {', + ' pub(crate) fn unmap_heap_above(&self, keep: usize) {')) start = pools.index(POOLS_NEW_BACKING[0]) - end = pools.index('\n', pools.index(POOLS_NEW_BACKING[1], start)) + 1 + end = pools.index(POOLS_NEW_BACKING[1], start) pools_new_backing = pools[start:end] if pools.count(PB_LEAF_ASSERT) != 1: raise SystemExit('pools.rs does not assert the PB leaf GPU-only') @@ -1424,32 +2100,49 @@ def run(mutation): if init.count(line) != 1: raise SystemExit('Pools::init() does not hold: ' + line) reset = '\n'.join('p.' + line[len('self.'):] for line in LEDGER_RESET) - pb_max_blocks = item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> u32 {') + pb_max_blocks = '\n'.join(item(src[PARAMS], f'pub(crate) fn {name}() -> ') + for name in ('pb_max_blocks', 'pb_blocks', 'pb_reclaim', 'pb_idle_ns', + 'pb_reclaim_blocks')) + reclaim_defaults = {} + for param, const in (('g15_pb_reclaim', 'PB_RECLAIM_DEFAULT'), ('g15_pb_idle_ms', 'PB_IDLE_MS_DEFAULT'), + ('g15_pb_reclaim_blocks', 'PB_RECLAIM_BLOCKS_DEFAULT')): + m = re.search(rf'{param}: u32 \{{\s*default: (\d+),', src[CORE]) + if not m: + raise SystemExit(f'{param} default not found') + reclaim_defaults[const] = int(m.group(1)) m = re.search(r'g15_pb_max_blocks: u32 \{\s*default: (\d+),', src[CORE]) if not m: raise SystemExit('g15_pb_max_blocks default not found') param_default = int(m.group(1)) + m = re.search(r'g15_pb_blocks: u32 \{\s*default: (\d+),', src[CORE]) + if not m: + raise SystemExit('g15_pb_blocks default not found') + blocks_default = int(m.group(1)) sizes = consts(item(hw, 'pub(crate) mod sizes {'), ['PB_BLOCKS']) - tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_BLOCKS', 'PB_MAX_BLOCKS']) + tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_MAX_BLOCKS']) client = consts(item(hw, 'pub(crate) mod client {'), ['PB_HEAP', 'PB_BLOCK_LIST']) pte = consts(item(hw, 'pub(crate) mod pte {'), ['LEAF_CACHED']) control = consts(item(src[INITDATA], 'pub(crate) mod control {'), ['HALT_COUNT']) growth = src[GROWTH] clamp_fn = item(growth, 'const fn clamp(v: u32, lo: u32, hi: u32) -> u32 {') - tested = (''.join(bodies) + soft_max + backed + chunk + pools_new_backing + pb_max_blocks + growth + + tested = (''.join(bodies) + soft_max + pools_methods + chunk + chunk_geometry + vm_methods + + pools_new_backing + pb_max_blocks + growth + src[BUFFER] + sizes + tuning + pte + prot) dir = out / (mutation or 'actual') dir.mkdir(exist_ok=True) for path in (GROWTH, BUFFER, CHANNELS): (dir / Path(path).name).write_text(src[path]) code = (PREFIX + f'pub const PARAM_DEFAULT: u32 = {param_default};\n' + + f'pub const PB_BLOCKS_DEFAULT: u32 = {blocks_default};\n' + + ''.join(f'pub const {k}: u32 = {v};\n' for k, v in reclaim_defaults.items()) + 'pub mod g15 {\n' ' pub mod fw {\n' f' #[path = "{dir / "channels.rs"}"] pub mod channels;\n' f' #[path = "{dir / "buffer.rs"}"] pub mod buffer;\n' ' pub mod initdata { pub mod control {\n' + control + '\n} }\n' ' }\n' - f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth; }}\n' + f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth;\n' + ' use super::fw::buffer;\n use super::hw::client;\n' + chunk_geometry + '\n }\n' ' pub mod hw {\n' ' pub mod sizes {\n' + sizes + '\n}\n' ' pub mod tuning {\n' + tuning + '\n}\n' @@ -1458,6 +2151,7 @@ def run(mutation): ' }\n' ' pub mod pgtable {\n' + pg_consts + '\n' + prot + '\n }\n' ' pub mod boot {\n pub fn eproto() -> crate::Error { crate::EPROTO }\n }\n' + ' pub mod engine {\n pub fn now_ns() -> u64 { crate::NOW.with(|n| n.get()) }\n }\n' ' /// The checked backing allocation, failing where a test injects it.\n' ' pub mod object {\n' ' use crate::prelude::*;\n' @@ -1482,19 +2176,20 @@ def run(mutation): ' use super::fw::buffer::{self, block_ctl};\n' ' use super::fw::channels::DevCtl;\n' ' use super::fw::initdata::control;\n' - ' use super::hw::{client, pte, tuning};\n' - ' use super::pools::growth::{self, Ledger};\n' + ' use super::hw::{self, client, pte, tuning};\n' + ' use super::pools::growth::{self, Answer, Ledger};\n' + ' use super::pools::{chunk_gva, CHUNK_BYTES};\n' ' use super::object::{self, HeldPages};\n' ' use crate::driver::AsahiDevice;\n' - ' use crate::{gem, mmu, ARef};\n' - ' use core::sync::atomic::{AtomicUsize, Ordering};\n' + ' use crate::{gem, mmu, ARef, Mutex};\n' ' use std::sync::Arc;\n' + MODEL.replace('p.RESET', reset).replace('POOLS_NEW_BACKING', '') + chunk + '\n' ' /// Pools::new\'s PB backing, verbatim.\n' ' pub fn pools_new_backing(dev: &AsahiDevice) -> Result<(Ledger, KVec)> {\n' + pools_new_backing + ' Ok((ledger, chunks))\n }\n' - ' impl Pools {\n' + soft_max + '\n' + backed + '\n }\n' + ' impl Pools {\n' + soft_max + '\n' + pools_methods + '\n }\n' + ' impl VmState {\n' + vm_methods + '\n }\n' ' impl Firmware {\n' + '\n'.join(bodies) + '\n }\n' ' }\n' '}\n' + TESTS) @@ -1534,7 +2229,8 @@ def no_core(): result = run(a.mutation) receipt = {'ref': a.ref, 'old_ref': OLD_REF, 'old_reference_checked_against_git': old_git is not None, **result, - 'scope': 'Actual answer_growth/answer_request/soft_max/pb_max_blocks, pools/growth.rs, ' + 'scope': 'Actual answer_growth/answer_request/map_pb/refill_backing/soft_max/pb_max_blocks, ' + 'the PB chunk and VM heap-mapping bodies, pools/growth.rs, ' 'fw/buffer.rs and fw/channels.rs over a model of the firmware objects with an ' 'ordered access log, and Ledger::new against its body at ' f'{OLD_REF[:12]}. Not kernel, firmware or GPU execution, timing, or recovery.'} diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index 9acd1f012413ad..42b55f063359eb 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -174,6 +174,8 @@ def item(text, marker): mod boot { pub mod glue { pub fn retained() -> bool { crate::RETAINED.with(core::cell::Cell::get) } pub unsafe fn asahi_g15_reprobe() { crate::activation_reprobe(); } + // Reads the device tree and logs; no driver state. + pub unsafe fn asahi_g15_report_loader_gpu() {} }} ''' diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index 5b728e95fc03e2..e659a507477b56 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -5,8 +5,10 @@ Uses synthetic public data and temporary host copies of production Rust modules. No GPU, firmware file, private fixture, kernel build or hardware access. The golden T6030 inputs below are from SiliconForest 9f09c85f8d3b, except the full -hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes. -Its T6030 configuration module is independently pinned to the original bytes. +hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes +and again when the initial PB posting became a module parameter (the tuning +constant PB_BLOCKS removed). Its T6030 configuration module is independently +pinned to the original bytes. """ import argparse @@ -22,7 +24,7 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "e4619590e670ed88d2a9bb76b03853081c615d3d0ad1c6a68321ce8e615ef8a0", + "hw.rs": "658c58026553c098b899aeffa0b6ae5be2b14921aef19df9e4f14789b7b890c4", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index 5ac596625d9acc..2ab89ebfc7d5db 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -554,7 +554,7 @@ def publication(root): state = n['state'] state = state.replace('struct apple_connector {', '''struct drm_device { void *dev; struct { bool mutex; } mode_config; }; struct drm_connector { struct drm_device *dev; int registration_state; void *state; - struct { struct { unsigned min_vfreq,max_vfreq; } monitor_range; } display_info; + struct { struct { unsigned min_vfreq,max_vfreq; } monitor_range; int width_mm,height_mm; } display_info; }; struct apple_connector {''') state = state.replace(' struct { int registration_state; } base;', ' struct drm_connector base;') @@ -569,7 +569,7 @@ def publication(root): int fw_compat,width_mm,height_mm,fixed_connector_type; unsigned notch_height; bool modes_lock,modes_admitted,modes_provisional,placeholder_retried,external_native; int external_held; u64 modes_generation,dimensions_generation; struct work_struct dimensions_wq; - struct { int width_mm,height_mm; } panel; + struct { int width_mm,height_mm; struct { char vendor[4]; } id; } panel; struct { size_t length,received; void *data; bool contiguous,payload_complete; u64 modes_generation; u64 av_service_epoch,automatic_connection_sequence; bool automatic_service_pending; struct dcp_metadata_ticket metadata_ticket,mode_ticket; @@ -612,6 +612,11 @@ def publication(root): static bool cancel_work_sync(struct work_struct *w) { assert(!w->scheduled); return false; } /* Native audio is out of scope: reject entry instead of silently queueing it. */ static void dcp_external_audio_start(struct apple_dcp *d) { assert(!d->external_native); } +/* These H17P scenarios never reach the 14.7 panel's base EDID. */ +#define dev_warn_once(...) ((void)0) +static bool dcp_is_v14_7_panel(const struct apple_dcp *d) { (void)d; return false; } +static int dcp_panel_edid_update(struct drm_connector *c,const void *id,int w,int h) { (void)c;(void)id;(void)w;(void)h;assert(false);return 0; } +static void dcp_update_connector_dimensions(struct apple_dcp *d,struct drm_connector *c) { (void)d;(void)c;assert(false); } static void dpavservep_detach(struct apple_dcp *d); static struct apple_dcp *probing; static struct drm_edid transport_edid; diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 35c96a7f36176c..861290ef240986 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -102,6 +102,7 @@ pub type Result = std::result::Result; pub const EINVAL: Error = -22; pub const EFAULT: Error = -14; + pub const ENOSPC: Error = -28; pub use crate::{pr_warn, static_assert}; } #[macro_export] @@ -124,8 +125,19 @@ } mod command { use crate::{mmu, prelude::*}; + use std::cell::Cell; pub const USC_WINDOW: u64 = 1 << 32; + /// Addresses from here on are mapped and readable without backing in the mock VM. + pub const SYNTHETIC: u64 = 1 << 40; + thread_local! { + /// cover() calls: the page checks the walk made. + pub static COVERS: Cell = Cell::new(0); + } pub fn cover(vm: &mmu::Vm, start: u64, end: u64, _: bool, _: bool) -> Result { + COVERS.with(|c| c.set(c.get() + 1)); + if start >= SYNTHETIC { + return Ok(()); + } let mut bytes = vec![0; usize::try_from(end - start).map_err(|_| EINVAL)?]; vm.read_user(start, &mut bytes) } @@ -151,16 +163,88 @@ fn cyclic_links_are_rejected() { assert_eq!(gate::cdm(&vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); } + /// Words of 4 bytes that fill the byte budget: the root block and one per link. + const BUDGET_WORDS: usize = 65 * 0x10000 / 4; #[test] - fn record_budget_requires_a_terminator() { - assert_eq!(gate::cdm(&vm(&vec![0x60000160; 8192]), 0, 0), Err(EINVAL)); + fn byte_budget_requires_a_terminator() { + assert_eq!(gate::cdm(&vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); } #[test] fn final_record_may_terminate() { - let mut words = vec![0x60000160; 8192]; - words[8191] = 0x40000000; + let mut words = vec![0x60000160; BUDGET_WORDS + 1]; + words[BUDGET_WORDS - 1] = 0x40000000; + assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + words[BUDGET_WORDS - 1] = 0x60000160; + words[BUDGET_WORDS] = 0x40000000; + assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + } + #[test] + fn more_than_8192_records_are_admitted() { + // 1024 draws of ten records each, as a tessellated multi-draw writes them. + let mut words = vec![0x60000160; 10240]; + words.push(0x40000000); + assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + } + #[test] + fn link_budget_still_bounds_the_walk() { + // 65 links, each to the next: the 65th is refused. + let mut words = Vec::new(); + for i in 0..65u32 { + words.extend([0x20000000, (i + 1) * 8]); + } + words.push(0x40000000); + assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + let mut words = Vec::new(); + for i in 0..64u32 { + words.extend([0x20000000, (i + 1) * 8]); + } + words.push(0x40000000); assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); } + /// The USC window of these cases and its first program, both in the mock's mapped range. + const USC: u64 = crate::command::SYNTHETIC; + /// A 24-byte indirect launch of the window's first program with its arguments at `args`. + fn indirect(args: u64) -> [u32; 6] { + [0x10080000, ((USC >> 38) as u32) << 22, (USC >> 6) as u32, + 0x40000000 | ((USC >> 40) & 0xff) as u32, (args >> 32) as u32 & 0xffff, args as u32] + } + /// `n` indirect launches, launch `i` with its arguments at `args(i)`, then TERMINATE. + fn launches(n: usize, args: impl Fn(usize) -> u64) -> Vec { + let mut words: Vec = (0..n).flat_map(|i| indirect(args(i))).collect(); + words.push(0x40000000); + words + } + fn covers(words: &[u32]) -> (Result, u32) { + crate::command::COVERS.with(|c| c.set(0)); + let r = gate::cdm(&vm(words), USC, 0); + (r, crate::command::COVERS.with(|c| c.get())) + } + const PAGE: u64 = mmu::UAT_PGSZ as u64; + #[test] + fn repeated_argument_pages_are_checked_once() { + // 1000 launches with their arguments in one page: the program page and that page. + let base = 2 * USC; + assert_eq!(covers(&launches(1000, |i| base + (i as u64 % 100) * 12)), (Ok(()), 2)); + // Alternating between two pages: one check per change of page. + assert_eq!(covers(&launches(10, |i| base + (i as u64 % 2) * PAGE)), (Ok(()), 11)); + // Arguments that cross a page are checked each time. + assert_eq!(covers(&launches(3, |_| base + PAGE - 8)), (Ok(()), 4)); + } + #[test] + fn page_check_budget_bounds_the_walk() { + let base = 2 * USC; + // The program page and 16383 argument pages: 16384 checks, admitted. + assert_eq!(covers(&launches(16383, |i| base + i as u64 * PAGE)), (Ok(()), 16384)); + // One more page: refused at the 16385th check, which is not made. + assert_eq!(covers(&launches(16384, |i| base + i as u64 * PAGE)), (Err(EINVAL), 16384)); + } + #[test] + fn a_large_stream_of_reused_pages_is_admitted() { + // 170000 launches (4 MB, inside the byte budget) over 16 argument pages. + let base = 2 * USC; + let words = launches(170000, |i| base + (i as u64 / 10625) * PAGE + (i as u64 % 1000) * 12); + assert_eq!(covers(&words), (Ok(()), 17)); + } #[test] fn malformed_and_unreadable_streams_are_rejected() { assert_eq!(gate::cdm(&vm(&[0xffffffff]), 0, 0), Err(EINVAL));