From 0fb022b7e5350788bdf58cbd993d77d1fb8e9968 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Thu, 19 Nov 2026 06:49:23 +0100 Subject: [PATCH 1/5] accel/ane: h16: give back the command arena of unloaded programs The transport copies every command into a 256 KiB pool arena. Each EXECUTE sequence gives its space back once the firmware reported the request complete, but LOAD, CREATE, TERMINATE, UNLOAD and the gating commands outside a request stay until the next firmware boot: 1164 bytes per program loaded and closed. About 210 programs into one firmware boot, h16_pool_room() refuses every LOAD with -ENOSPC. On H17 an idle park or a system sleep boots the firmware again and starts a new arena; a client that keeps the engine busy, or a platform that does not park, runs into it. Take the arena position at a LOAD that finds no program alive in this firmware boot. When the last program is torn down cleanly, with nothing in flight and the session not quarantined, the firmware is done with every command published since, so release the arena to that position. A program still loaded keeps its commands and every later one; a new firmware boot drops the position. Assisted-by: LLM Signed-off-by: Bartosz Nitka --- drivers/accel/ane/ane_h16_program.c | 29 +++++++++++- drivers/accel/ane/ane_h16_program.h | 6 +++ drivers/accel/ane/ane_h16_program_test.c | 58 ++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/drivers/accel/ane/ane_h16_program.c b/drivers/accel/ane/ane_h16_program.c index 9132cc2ddaded..294646d14584e 100644 --- a/drivers/accel/ane/ane_h16_program.c +++ b/drivers/accel/ane/ane_h16_program.c @@ -597,8 +597,9 @@ static int h16_send(struct ane_h16_session *s, unsigned int ch, const void *comm /* * The transport copies commands into a pool arena. LOAD and CREATE stay in - * it; each EXECUTE sequence gives its space back once the firmware reported - * the request complete. Refuse a sequence up front when it cannot fit, so + * it while any program of this firmware boot is alive (h16_pool_rewind()); + * each EXECUTE sequence gives its space back once the firmware reported the + * request complete. Refuse a sequence up front when it cannot fit, so * running out of arena is a clean -ENOSPC and never a half-sent sequence. */ static int h16_pool_room(struct ane_h16_session *s, const size_t *lengths, @@ -617,6 +618,23 @@ static int h16_pool_room(struct ane_h16_session *s, const size_t *lengths, return 0; } +/* + * The last program of this firmware boot was torn down cleanly and nothing + * is in flight: the firmware is done with every command published since the + * LOAD that found no program alive (LOAD, CREATE, gates, TERMINATE and + * UNLOAD), so give their arena back. Without this the arena filled after + * about two hundred programs and every LOAD failed with -ENOSPC until the + * firmware was booted again. + */ +static void h16_pool_rewind(struct ane_h16_session *s) +{ + if (!s->pool_base_set || !list_empty(&s->programs) || !list_empty(&s->held) || + s->running || s->active || s->quarantined) + return; + if (!ane_fw_transport_release(s->transport, s->pool_base)) + s->pool_base_set = false; +} + static void h16_sync(struct ane_h16_session *s, const struct ane_fw_surface *surface, bool for_device) { @@ -995,6 +1013,10 @@ int ane_h16_session_load(struct ane_h16_session *s, p->closed = true; goto out; } + if (list_empty(&s->programs) && list_empty(&s->held) && !s->running) { + s->pool_base = ane_fw_transport_mark(s->transport); + s->pool_base_set = true; + } ret = h16_send(s, ANE_FW_IO, command, ANE_H16_LOAD_SIZE, &reply); if (!ret) ret = ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE, command, &p->key.program); @@ -1772,6 +1794,7 @@ int ane_h16_session_close(struct ane_h16_session *s, struct ane_h16_program *p) * reads or writes this program's memory: later programs may reuse it. */ h16_give_back(s, p); + h16_pool_rewind(s); return 0; } EXPORT_SYMBOL_GPL(ane_h16_session_close); @@ -1850,6 +1873,8 @@ void ane_h16_session_restart(struct ane_h16_session *s) { s->epoch++; s->program = NULL; + /* The new firmware's arena starts afresh. */ + s->pool_base_set = false; /* A firmware booted again from the preload starts with gating on. */ s->gate_held = false; s->executed = 0; diff --git a/drivers/accel/ane/ane_h16_program.h b/drivers/accel/ane/ane_h16_program.h index c22919a79349c..83c87480612d0 100644 --- a/drivers/accel/ane/ane_h16_program.h +++ b/drivers/accel/ane/ane_h16_program.h @@ -349,6 +349,12 @@ struct ane_h16_session { bool pipelined; /* Reclaim a batch of command buffers only after its last caller ends. */ size_t execute_mark; + /* + * Command arena position at a LOAD with no program alive in this + * firmware boot: once none is again, every command above it is done. + */ + size_t pool_base; + bool pool_base_set; /* Firmware log lines printed after the last failed EXECUTE. */ unsigned int fwlog_lines; bool quarantined; diff --git a/drivers/accel/ane/ane_h16_program_test.c b/drivers/accel/ane/ane_h16_program_test.c index cf494e1a1c0ea..177e17785211e 100644 --- a/drivers/accel/ane/ane_h16_program_test.c +++ b/drivers/accel/ane/ane_h16_program_test.c @@ -2809,7 +2809,65 @@ static void program_hung_recovery_test(struct kunit *test) ane_h16_session_put(&f->s, p); } +/* Load, run, close and free one program. */ +static int fw_cycle(struct prog_fw *f) +{ + struct ane_h16_program *p = NULL; + int ret = fw_load_program(f, &p); + + if (ret) + return ret; + ret = ane_h16_session_execute(&f->s, p); + if (!ret) + ret = ane_h16_session_close(&f->s, p); + if (!ret) + ane_h16_session_put(&f->s, p); + return ret; +} + +/* + * LOAD, CREATE, TERMINATE and UNLOAD of each program used to stay in the + * 256 KiB command arena until the firmware was booted again: about two + * hundred programs in, every LOAD failed with -ENOSPC. Once the last + * program is torn down the arena goes back to where the first LOAD found + * it, and never below a program that is still loaded. + */ +static void program_pool_rewind_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + size_t base, kept; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + base = ane_fw_transport_mark(&f->t); + for (i = 0; i < 1000; i++) { + KUNIT_ASSERT_EQ(test, fw_cycle(f), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_mark(&f->t), base); + } + + /* A program that stays loaded keeps its commands and all after them. */ + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + kept = ane_fw_transport_mark(&f->t); + KUNIT_EXPECT_GT(test, kept, base); + for (i = 0; i < 20; i++) + KUNIT_ASSERT_EQ(test, fw_cycle(f), 0); + KUNIT_EXPECT_GT(test, ane_fw_transport_mark(&f->t), kept); + KUNIT_ASSERT_EQ(test, ane_h16_session_close(&f->s, p), 0); + ane_h16_session_put(&f->s, p); + KUNIT_EXPECT_EQ(test, ane_fw_transport_mark(&f->t), base); + + /* A quarantined session keeps the arena as it is. */ + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + kept = ane_fw_transport_mark(&f->t); + f->s.quarantined = true; + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, p), -EIO); + KUNIT_EXPECT_EQ(test, ane_fw_transport_mark(&f->t), kept); + KUNIT_EXPECT_FALSE(test, list_empty(&f->s.held)); +} + static struct kunit_case program_cases[] = { + KUNIT_CASE_SLOW(program_pool_rewind_test), KUNIT_CASE(program_hung_recovery_test), KUNIT_CASE(program_pipeline_out_of_order_test), KUNIT_CASE(program_pipeline_same_program_test), From 4883364918678d211b39faabb39a4e2ee626e15c Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Thu, 19 Nov 2026 06:49:37 +0100 Subject: [PATCH 2/5] accel/ane: h16: free spare program memory through the backend Program memory never goes back before reset. A closed program's surfaces become spares, which serve only a later mapping of the same size and permission; the table holds 64, and anything beyond it, or a LOAD that failed before or at the firmware, keeps its memory mapped and unused. A client that loads programs of many shapes (a sweep of held-out conv layers, camera frames at several sizes) thus fills the backend's DVA window: on the J700, 337 programs into a boot, every H16_LOAD failed with -ENOSPC until reboot, and the memory behind the window stayed allocated too. Add an optional ops->unmap that unmaps a surface, frees its memory and its DVA range, and use it for memory the firmware no longer uses: - Keep the spares oldest first, and free the oldest beyond 32 entries or 128 MiB after every give-back and LOAD. A spare table that is full frees the surface instead of keeping it. - When a mapping finds no room (-ENOSPC or -ENOMEM), free every spare and try once more. - A LOAD that fails before the firmware saw the program gives its memory back and frees the program. One the firmware refused, at LOAD or CREATE, joins the programs held for the verified power cycle of the recovery, which gives them back. The DART invalidation of an unmap must reach the engine, so nothing is freed while the session is asleep or parked; the spares keep it until the next LOAD, which runs with the engine up. Every caller holds the session lock. Without ops->unmap nothing changes. Assisted-by: LLM Signed-off-by: Bartosz Nitka --- drivers/accel/ane/ane_h16_program.c | 94 +++++++-- drivers/accel/ane/ane_h16_program.h | 26 ++- drivers/accel/ane/ane_h16_program_test.c | 247 +++++++++++++++++++++++ 3 files changed, 347 insertions(+), 20 deletions(-) diff --git a/drivers/accel/ane/ane_h16_program.c b/drivers/accel/ane/ane_h16_program.c index 294646d14584e..d6a72ef64ec79 100644 --- a/drivers/accel/ane/ane_h16_program.c +++ b/drivers/accel/ane/ane_h16_program.c @@ -658,6 +658,37 @@ void ane_h16_session_sync(struct ane_h16_session *s, struct ane_h16_program *p, } EXPORT_SYMBOL_GPL(ane_h16_session_sync); +/* + * Spare memory can go back to the backend: it has ops->unmap and the engine + * is up, so the DART invalidation that comes with the unmap reaches it. + */ +static bool h16_can_unmap(const struct ane_h16_session *s) +{ + return s->ops->unmap && !s->asleep && !s->parked; +} + +static struct ane_fw_surface h16_spare_remove(struct ane_h16_session *s, unsigned int i) +{ + struct ane_fw_surface surface = s->spares[i].surface; + + s->spare_bytes -= surface.size; + s->spare_count--; + memmove(&s->spares[i], &s->spares[i + 1], (s->spare_count - i) * sizeof(s->spares[0])); + return surface; +} + +/* Free the oldest spares until at most @count entries and @bytes remain. */ +static void h16_spares_trim(struct ane_h16_session *s, unsigned int count, size_t bytes) +{ + if (!h16_can_unmap(s)) + return; + while (s->spare_count && (s->spare_count > count || s->spare_bytes > bytes)) { + struct ane_fw_surface surface = h16_spare_remove(s, 0); + + s->ops->unmap(s->ctx, &surface); + } +} + static bool h16_take_spare(struct ane_h16_session *s, size_t size, bool write, struct ane_fw_surface *out) { @@ -666,8 +697,7 @@ static bool h16_take_spare(struct ane_h16_session *s, size_t size, bool write, for (i = 0; i < s->spare_count; i++) { if (s->spares[i].surface.size != size || s->spares[i].write != write) continue; - *out = s->spares[i].surface; - s->spares[i] = s->spares[--s->spare_count]; + *out = h16_spare_remove(s, i); /* Zeroed like new memory; LOAD writes it back after filling it. */ memset(out->cpu, 0, size); return true; @@ -678,12 +708,20 @@ static bool h16_take_spare(struct ane_h16_session *s, size_t size, bool write, static void h16_give_spare(struct ane_h16_session *s, struct ane_fw_surface *surface, bool write) { - /* A full table keeps the surface mapped and unused until reset. */ - if (surface->cpu && s->spare_count < ANE_H16_SPARES) + if (!surface->cpu) { + memset(surface, 0, sizeof(*surface)); + return; + } + if (s->spare_count < ANE_H16_SPARES) { s->spares[s->spare_count++] = (struct ane_h16_spare) { .surface = *surface, .write = write, }; + s->spare_bytes += surface->size; + } else if (h16_can_unmap(s)) { + s->ops->unmap(s->ctx, surface); + } + /* Otherwise a full table keeps the surface mapped and unused until reset. */ memset(surface, 0, sizeof(*surface)); } @@ -702,6 +740,7 @@ static void h16_export_work(struct work_struct *work) h16_give_spare(s, &e->surface, e->write); kfree(e); } + h16_spares_trim(s, ANE_H16_SPARES_KEEP, ANE_H16_SPARE_BYTES); mutex_unlock(s->lock); } @@ -793,13 +832,22 @@ static void h16_give_back(struct ane_h16_session *s, struct ane_h16_program *p) h16_give_spare(s, &p->aux[i], false); memset(p->surfaces, 0, sizeof(p->surfaces)); p->released = true; + h16_spares_trim(s, ANE_H16_SPARES_KEEP, ANE_H16_SPARE_BYTES); } static int h16_map(struct ane_h16_session *s, size_t size, bool write, struct ane_fw_surface *out) { - int ret = h16_take_spare(s, size, write, out) ? 0 : s->ops->map(s->ctx, size, write, out); + int ret = 0; + if (!h16_take_spare(s, size, write, out)) { + ret = s->ops->map(s->ctx, size, write, out); + /* No DVA range or memory left: free every spare and try once more. */ + if ((ret == -ENOSPC || ret == -ENOMEM) && s->spare_count && h16_can_unmap(s)) { + h16_spares_trim(s, 0, 0); + ret = s->ops->map(s->ctx, size, write, out); + } + } if (ret) return ret; if (!ane_fw_surface_valid(out) || out->size != size || !IS_ALIGNED(out->dva, SZ_16K)) @@ -922,6 +970,19 @@ static int h16_program_map(struct ane_h16_session *s, struct ane_h16_program *p, return ane_h16_custom_bars(&p->snapshot, hwx, s->dsid, p->bars, &p->bar_count); } +/* + * LOAD or CREATE failed with the firmware holding the program, maybe: the + * session is quarantined and the program is kept, memory and all, with + * those closed while quarantined, until a verified power cycle gives them + * back (ane_h16_session_recovered()). + */ +static int h16_failed_load(struct ane_h16_session *s, struct ane_h16_program *p, int error) +{ + p->closed = true; + list_add_tail(&p->node, &s->held); + return h16_fail(s, error); +} + static const size_t h16_load_sequence[] = { ANE_H16_LOAD_SIZE, ANE_H16_CREATE_SIZE, ANE_H16_TERMINATE_SIZE, ANE_H16_UNLOAD_SIZE, }; @@ -933,8 +994,9 @@ static const size_t h16_load_sequence[] = { * queues the first EXECUTE whenever the engine last ran another program, the * first one after boot included (see ane_h16_session_execute()). A LOAD is * serialized with in-flight EXECUTEs. The transport also requeues on every - * program or process change, so a session may hold several programs. Every - * program's firmware-visible memory stays mapped until reset. + * program or process change, so a session may hold several programs. A + * program's firmware-visible memory goes back to the spares once the + * firmware no longer uses it (see h16_give_back()). */ int ane_h16_session_load(struct ane_h16_session *s, struct ane_h16_hwx_snapshot *snapshot, @@ -991,9 +1053,9 @@ int ane_h16_session_load(struct ane_h16_session *s, if (ret) goto free; /* - * From the first mapping on, firmware-visible memory exists that is never - * released before reset, so the program becomes the session's here and - * is never freed. + * From the first mapping on, firmware-visible memory exists, so the + * program becomes the session's here; it is freed only once that memory + * went back (ane_h16_session_put()). */ p->generation = ++s->generation; p->epoch = s->epoch; @@ -1006,11 +1068,10 @@ int ane_h16_session_load(struct ane_h16_session *s, if (!ret) ret = ane_h16_load_encode(command, ANE_H16_LOAD_SIZE, p->slots); if (ret) { - /* - * Firmware never saw this program. It keeps its mappings until - * reset, but it must not hold the session against the next LOAD. - */ + /* Firmware never saw this program: its memory goes back now. */ p->closed = true; + h16_give_back(s, p); + ane_h16_session_put(s, p); goto out; } if (list_empty(&s->programs) && list_empty(&s->held) && !s->running) { @@ -1022,7 +1083,7 @@ int ane_h16_session_load(struct ane_h16_session *s, ret = ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE, command, &p->key.program); if (ret) { s->ops->log(s->ctx, "LOAD refused", (u64)-ret, 0); - ret = h16_fail(s, ret); + ret = h16_failed_load(s, p, ret); goto out; } s->ops->log(s->ctx, "LOAD program", p->key.program, p->bar_count); @@ -1034,7 +1095,7 @@ int ane_h16_session_load(struct ane_h16_session *s, &p->key.process); if (ret) { s->ops->log(s->ctx, "CREATE refused", (u64)-ret, 0); - ret = h16_fail(s, ret); + ret = h16_failed_load(s, p, ret); goto out; } /* Each validated program has exactly one procedure. */ @@ -1044,6 +1105,7 @@ int ane_h16_session_load(struct ane_h16_session *s, s->ops->log(s->ctx, "CREATE process", p->key.program, p->key.process); *program = p; out: + h16_spares_trim(s, ANE_H16_SPARES_KEEP, ANE_H16_SPARE_BYTES); kfree(command); return ret; free: diff --git a/drivers/accel/ane/ane_h16_program.h b/drivers/accel/ane/ane_h16_program.h index 83c87480612d0..d4ca6a37e3023 100644 --- a/drivers/accel/ane/ane_h16_program.h +++ b/drivers/accel/ane/ane_h16_program.h @@ -150,10 +150,22 @@ int ane_h16_custom_bars(const struct ane_h16_hwx_snapshot *snapshot, const struct ane_h16_hwx *hwx, u32 dsid, struct ane_h16_custom_bar *bars, unsigned int *count); -/* Private, pinned mappings in the firmware domain. Never released before reset. */ +/* + * Private, pinned mappings in the firmware domain. Without unmap they are + * never released before reset. + */ struct ane_h16_session_ops { int (*map)(void *ctx, size_t size, bool device_write, struct ane_fw_surface *surface); + /* + * Optional. Give back a surface map() returned: unmap it from the + * engine, free its memory and its DVA range. The session calls it + * only for memory the firmware no longer uses (UNLOAD acknowledged, + * the program lost to a new firmware, or never seen by the firmware), + * only while the engine is up (not asleep, not parked), and with the + * session lock held. + */ + void (*unmap)(void *ctx, const struct ane_fw_surface *surface); void (*log)(void *ctx, const char *item, u64 a, u64 b); /* * Optional. For a surface with a handle: before the engine uses it @@ -286,11 +298,15 @@ void ane_h16_exec_stats_add(struct ane_h16_exec_stats *st, */ /* * Program memory given back by a program that the firmware unloaded. It - * stays mapped until reset like every firmware-visible mapping, and serves - * a later program's mapping of the same size and permission instead of a - * new one, so loading the same model again and again uses no new memory. + * serves a later program's mapping of the same size and permission instead + * of a new one, so loading the same model again and again uses no new + * memory. With ops->unmap the oldest spares beyond ANE_H16_SPARES_KEEP + * entries or ANE_H16_SPARE_BYTES are freed, and all of them when a new + * mapping finds no room; without it, they stay mapped until reset. */ #define ANE_H16_SPARES 64 +#define ANE_H16_SPARES_KEEP 32 +#define ANE_H16_SPARE_BYTES SZ_128M struct ane_h16_spare { struct ane_fw_surface surface; @@ -305,8 +321,10 @@ struct ane_h16_session { struct ane_h16_pending_table pending; /* The last loaded program. */ struct ane_h16_program *program; + /* Oldest first. */ struct ane_h16_spare spares[ANE_H16_SPARES]; unsigned int spare_count; + size_t spare_bytes; /* Quiet time that ends serving the SHAREDMALLOC ring after UNLOAD. */ u64 settle_ns; /* diff --git a/drivers/accel/ane/ane_h16_program_test.c b/drivers/accel/ane/ane_h16_program_test.c index 177e17785211e..04be0bc362dff 100644 --- a/drivers/accel/ane/ane_h16_program_test.c +++ b/drivers/accel/ane/ane_h16_program_test.c @@ -80,6 +80,13 @@ struct prog_fw { /* ops->hung calls and the error of the last one. */ unsigned int hung; int hung_error; + /* + * Bytes mapped and not unmapped, ops->unmap calls, the most bytes a + * nonzero window lets be mapped at once (-ENOSPC beyond), and the + * number of the map call that fails with -ENOMEM (0: none). + */ + u64 live, window; + unsigned int unmaps, map_fail_at; }; static u64 fw_load(void *ctx, const void *p) @@ -328,6 +335,12 @@ static int fw_map(void *ctx, size_t size, bool write, struct ane_fw_surface *s) { struct prog_fw *f = ctx; + if (f->window && size > f->window - f->live) + return -ENOSPC; + if (f->map_fail_at && f->maps + 1 == f->map_fail_at) { + f->map_fail_at = 0; + return -ENOMEM; + } *s = (struct ane_fw_surface) { .cpu = fw_zalloc(f->test, size), .dva = f->next_dva, @@ -339,9 +352,21 @@ static int fw_map(void *ctx, size_t size, bool write, struct ane_fw_surface *s) f->next_dva += size; f->maps++; f->writable_maps += write; + f->live += size; return 0; } +static void fw_unmap(void *ctx, const struct ane_fw_surface *s) +{ + struct prog_fw *f = ctx; + + KUNIT_EXPECT_FALSE(f->test, f->s.asleep || f->s.parked); + KUNIT_EXPECT_GE(f->test, f->live, (u64)s->size); + f->live -= s->size; + f->unmaps++; + kunit_release_action(f->test, fw_kvfree, s->cpu); +} + static void fw_log(void *ctx, const char *item, u64 a, u64 b) { struct prog_fw *f = ctx; @@ -399,6 +424,16 @@ static void fw_hung(void *ctx, int error) f->hung_error = error; } +/* As fw_session_ops, with a backend that frees what the session gives back. */ +static const struct ane_h16_session_ops fw_session_ops_unmap = { + .map = fw_map, + .unmap = fw_unmap, + .log = fw_log, + .busy = fw_busy, + .sync = fw_sync, + .hung = fw_hung, +}; + /* As fw_session_ops, with a backend that recovers a quarantined session. */ static const struct ane_h16_session_ops fw_session_ops_hung = { .map = fw_map, @@ -2866,8 +2901,220 @@ static void program_pool_rewind_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, list_empty(&f->s.held)); } +/* + * Memory a program of another shape gave back: no later program of these + * tests maps a surface of this size, so only freeing it makes room. + */ +static bool fw_foreign_spare(struct prog_fw *f, size_t size) +{ + struct ane_fw_surface surface; + + if (f->s.spare_count == ANE_H16_SPARES || fw_map(f, size, false, &surface)) + return false; + f->s.spares[f->s.spare_count++] = (struct ane_h16_spare) { .surface = surface }; + f->s.spare_bytes += size; + return true; +} + +/* Larger than any region or section of the fixture programs. */ +#define FW_FOREIGN(i) (SZ_16K * (9 + (i) % 3)) +#define FW_WINDOW SZ_16M + +/* + * Programs of ever new shapes leave spares that no later program can use. + * Without ops->unmap a spare table full of them makes every close leave its + * memory mapped and unused, the DVA window fills, and every LOAD fails with + * -ENOSPC from then on, until reset. With it, the session frees the oldest + * spares beyond its limits: thousands of programs load into a window that + * holds a few dozen, with a new shape left behind before each. + */ +static void program_unmap_window_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test), *g = fw_create(test); + unsigned int i; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, g); + f->window = FW_WINDOW; + for (i = 0; i < ANE_H16_SPARES; i++) + KUNIT_ASSERT_TRUE(test, fw_foreign_spare(f, FW_FOREIGN(i))); + for (i = 0; i < 2000 && !ret; i++) + ret = fw_cycle(f); + KUNIT_EXPECT_EQ(test, ret, -ENOSPC); + KUNIT_EXPECT_LT(test, i, 2000U); + KUNIT_EXPECT_EQ(test, f->unmaps, 0U); + + g->s.ops = &fw_session_ops_unmap; + g->window = FW_WINDOW; + for (i = 0; i < ANE_H16_SPARES; i++) + KUNIT_ASSERT_TRUE(test, fw_foreign_spare(g, FW_FOREIGN(i))); + for (i = 0; i < 2000; i++) { + fw_foreign_spare(g, FW_FOREIGN(i)); + KUNIT_ASSERT_EQ(test, fw_cycle(g), 0); + KUNIT_ASSERT_LE(test, g->s.spare_count, ANE_H16_SPARES_KEEP); + } + KUNIT_EXPECT_GE(test, g->unmaps, 2000U); + KUNIT_EXPECT_FALSE(test, g->s.quarantined); + KUNIT_EXPECT_PTR_EQ(test, g->s.program, NULL); + /* What the spares hold is all that stays mapped. */ + KUNIT_EXPECT_EQ(test, g->live, (u64)g->s.spare_bytes); +} + +/* A mapping that finds no room frees every spare and tries again. */ +static void program_unmap_pressure_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->s.ops = &fw_session_ops_unmap; + f->window = SZ_2M; + for (i = 0; fw_foreign_spare(f, FW_FOREIGN(i)); i++) + ; + /* Not even the smallest mapping fits beside them. */ + f->window = f->live; + KUNIT_ASSERT_GT(test, i, 0U); + KUNIT_ASSERT_LT(test, i, ANE_H16_SPARES_KEEP); + KUNIT_ASSERT_EQ(test, fw_cycle(f), 0); + KUNIT_EXPECT_EQ(test, f->unmaps, i); + KUNIT_EXPECT_EQ(test, f->live, (u64)f->s.spare_bytes); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); +} + +static u8 fw_dummy[SZ_16K]; + +/* The spares stay within their byte limit too, freeing the oldest first. */ +static void program_unmap_bytes_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + const size_t half = ANE_H16_SPARE_BYTES / 2; + u64 newest = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->s.ops = &fw_session_ops_unmap; + /* Large spares with no memory behind them; never taken, only freed. */ + for (i = 0; i < 3; i++) { + newest = f->next_dva; + f->s.spares[f->s.spare_count++] = (struct ane_h16_spare) { + .surface = { .cpu = fw_dummy, .dva = newest, .size = half }, + }; + f->s.spare_bytes += half; + f->next_dva += half; + f->live += half; + } + KUNIT_ASSERT_EQ(test, fw_cycle(f), 0); + KUNIT_EXPECT_LE(test, f->s.spare_bytes, (size_t)ANE_H16_SPARE_BYTES); + KUNIT_EXPECT_EQ(test, f->live, (u64)f->s.spare_bytes); + KUNIT_EXPECT_EQ(test, f->unmaps, 2U); + for (i = 0; i < f->s.spare_count; i++) + if (f->s.spares[i].surface.size == half) + KUNIT_EXPECT_EQ(test, f->s.spares[i].surface.dva, newest); +} + +/* + * Nothing is freed while the engine is parked: the DART invalidation of an + * unmap could not reach it. The spares keep the memory until the next LOAD, + * which runs with the engine up. + */ +static void program_unmap_parked_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->s.ops = &fw_session_ops_unmap; + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + for (i = 0; i < ANE_H16_SPARES_KEEP + 8; i++) + KUNIT_ASSERT_TRUE(test, fw_foreign_spare(f, SZ_16K * 3)); + f->s.parked = true; + KUNIT_ASSERT_EQ(test, ane_h16_session_close(&f->s, p), 0); + ane_h16_session_put(&f->s, p); + KUNIT_EXPECT_EQ(test, f->unmaps, 0U); + KUNIT_EXPECT_GT(test, f->s.spare_count, ANE_H16_SPARES_KEEP); + f->s.parked = false; + KUNIT_ASSERT_EQ(test, fw_cycle(f), 0); + KUNIT_EXPECT_GT(test, f->unmaps, 0U); + KUNIT_EXPECT_LE(test, f->s.spare_count, ANE_H16_SPARES_KEEP); +} + +/* + * A LOAD that fails before the firmware saw the program (here, no memory + * for its third mapping) gives back what it mapped and frees the program. + */ +static void program_unmap_failed_map_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->s.ops = &fw_session_ops_unmap; + f->map_fail_at = 3; + KUNIT_EXPECT_EQ(test, fw_load_program(f, &p), -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, p, NULL); + KUNIT_EXPECT_PTR_EQ(test, f->s.program, NULL); + KUNIT_EXPECT_EQ(test, f->count, 0U); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); + KUNIT_EXPECT_EQ(test, f->s.spare_count, 2U); + KUNIT_EXPECT_EQ(test, f->live, (u64)f->s.spare_bytes); + /* The next LOAD takes them. */ + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + KUNIT_EXPECT_EQ(test, f->s.spare_count, 0U); +} + +/* + * A LOAD the firmware refused quarantines the session; the program is held + * with those closed while quarantined, and the verified power cycle of the + * recovery gives its memory back. + */ +static void program_unmap_refused_load_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + struct mutex *lock = kunit_kzalloc(test, sizeof(*lock), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, lock); + mutex_init(lock); + f->s.ops = &fw_session_ops_unmap; + f->mode = FW_LOAD_NAK; + KUNIT_EXPECT_NE(test, fw_load_program(f, &p), 0); + KUNIT_EXPECT_PTR_EQ(test, p, NULL); + KUNIT_EXPECT_TRUE(test, f->s.quarantined); + KUNIT_EXPECT_EQ(test, f->hung, 1U); + KUNIT_EXPECT_FALSE(test, list_empty(&f->s.held)); + KUNIT_EXPECT_EQ(test, f->s.spare_count, 0U); + + f->s.lock = lock; + ane_h16_session_sleep(&f->s, true); + fw_cold_boot(f); + /* The backend's cold boot starts a new transport. */ + memset(f->head, 0, sizeof(f->head)); + KUNIT_ASSERT_EQ(test, ane_fw_transport_init(&f->t, &f->soc, &fw_ops, f, f->ipc, f->pool, + SZ_16K, f->ipc.dva, ANE_FW_TABLE_SIZE), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + f->mode = FW_OK; + ane_h16_session_restart(&f->s); + ane_h16_session_recovered(&f->s); + ane_h16_session_sleep(&f->s, false); + KUNIT_EXPECT_TRUE(test, list_empty(&f->s.held)); + KUNIT_EXPECT_PTR_EQ(test, f->s.program, NULL); + KUNIT_EXPECT_GT(test, f->s.spare_count, 0U); + KUNIT_EXPECT_EQ(test, f->live, (u64)f->s.spare_bytes); + f->s.lock = NULL; + KUNIT_EXPECT_EQ(test, fw_cycle(f), 0); +} + static struct kunit_case program_cases[] = { KUNIT_CASE_SLOW(program_pool_rewind_test), + KUNIT_CASE_SLOW(program_unmap_window_test), + KUNIT_CASE(program_unmap_pressure_test), + KUNIT_CASE(program_unmap_bytes_test), + KUNIT_CASE(program_unmap_parked_test), + KUNIT_CASE(program_unmap_failed_map_test), + KUNIT_CASE(program_unmap_refused_load_test), KUNIT_CASE(program_hung_recovery_test), KUNIT_CASE(program_pipeline_out_of_order_test), KUNIT_CASE(program_pipeline_same_program_test), From de51a480fb9c73a160e4c12fab6fd6f8259d6a30 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Thu, 19 Nov 2026 06:49:44 +0100 Subject: [PATCH 3/5] accel/ane: h17: return program DVA ranges and memory The H17 backend hands out program and SHAREDMALLOC mappings by bumping a pointer through a 512 MiB DVA window, and never gives anything back: once the window is used up, every program mapping fails with -ENOSPC until reboot. On the J700 that happened after 337 program loads of a held-out layer sweep and some camera-denoise trials, and from then on every H16_LOAD failed, even of a 6 KB program that had run before. Allocate window ranges from a drm_mm instead, and implement the session's ops->unmap for program surfaces: find the mapping, unmap it from the DARTs (iommu_unmap() invalidates their TLBs before it returns), then free its pages and its range. A short unmap keeps the mapping, its memory and its range, as a failed mapping does. SHAREDMALLOC, pool, IPC, heap and preload mappings are never unmapped. A failed iommu_map() no longer leaves a partial mapping at the DVA the next allocation is given, since its range stays allocated with it. Assisted-by: LLM Signed-off-by: Bartosz Nitka --- drivers/accel/ane/ane_fw_cold.c | 112 ++++++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 19 deletions(-) diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index 2dc8d2239b89c..6aa6f1a29ad9a 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Shared firmware backend. H17 is the first measured cold power provider. */ #include +#include #include #include #include @@ -31,6 +32,10 @@ #define ANE_FW_IPC_DVA 0x100003d0000ULL #define ANE_FW_HEAP_DVA 0x100003e4000ULL #define ANE_FW_ALLOC_DVA 0x10001300000ULL +/* SHAREDMALLOC and program mappings: DVAs from ANE_FW_ALLOC_DVA on. */ +#define ANE_FW_WINDOW SZ_512M +/* ane_fw_alloc_mem(): a DVA range from the window. */ +#define ANE_FW_WINDOW_ANY U64_MAX #define ANE_FW_TIMEOUT_NS (3ULL * NSEC_PER_SEC) enum ane_fw_stage { @@ -47,6 +52,8 @@ struct ane_fw_mapping { struct list_head list; struct ane_fw_surface surface; dma_addr_t physical; + /* The DVA range, for a mapping in the window. */ + struct drm_mm_node node; }; struct ane_fw_backend { @@ -113,7 +120,13 @@ struct ane_fw_backend { u64 sys_off_at, sys_off_ns; struct ane_fw_cold_state cold; u64 ranges[ANE_FW_DAPF_MAX * 2]; - u64 next_dva; + /* + * The DVA window and the mappings list. Program mappings come and go + * under the session lock while the engine is up; the others are made + * with the session closed. + */ + struct drm_mm dva_window; + struct mutex window_lock; enum ane_fw_stage stage; bool retained; /* Brings the engine up to the program stage after a loader hand-off. */ @@ -1282,6 +1295,10 @@ static int ane_fw_alloc_mem(struct ane_fw_backend *b, u64 dva, size_t size, struct ane_fw_mapping *m; int ret; + /* No memory for a range the window cannot hold. */ + if (dva == ANE_FW_WINDOW_ANY && size > ANE_FW_WINDOW) + return -ENOSPC; + m = kzalloc_obj(*m); if (!m) return -ENOMEM; @@ -1301,10 +1318,32 @@ static int ane_fw_alloc_mem(struct ane_fw_backend *b, u64 dva, size_t size, dma_sync_single_for_device(b->memory, m->physical, size, DMA_BIDIRECTIONAL); m->surface.handle = m->physical; } + mutex_lock(&b->window_lock); + if (dva == ANE_FW_WINDOW_ANY) { + ret = drm_mm_insert_node_in_range(&b->dva_window, &m->node, size, SZ_16K, 0, + ANE_FW_ALLOC_DVA, + ANE_FW_ALLOC_DVA + ANE_FW_WINDOW, + DRM_MM_INSERT_LOW); + if (ret) { + mutex_unlock(&b->window_lock); + if (cached) + dma_free_noncoherent(b->memory, size, m->surface.cpu, + m->physical, DMA_BIDIRECTIONAL); + else + dma_free_coherent(b->memory, size, m->surface.cpu, m->physical); + kfree(m); + return ret; + } + dva = m->node.start; + } m->surface.dva = dva; m->surface.size = size; - /* Record before publication, including a failed/partial mapping. */ + /* + * Record before publication, including a failed/partial mapping: it + * keeps its memory and range until reset. + */ list_add_tail(&m->list, &b->mappings); + mutex_unlock(&b->window_lock); ret = ane_fw_map(b, dva, m->physical, size, prot); if (!ret) *s = m->surface; @@ -1322,14 +1361,11 @@ static int ane_fw_alloc(void *ctx, size_t size, u32 tag, struct ane_fw_surface * struct ane_fw_backend *b = ctx; int ret; - if (!size || size > ANE_FW_SHARED_TOTAL || - b->next_dva > ANE_FW_ALLOC_DVA + SZ_512M - size) + if (!size || size > ANE_FW_SHARED_TOTAL) return -ENOSPC; - ret = ane_fw_alloc_at(b, b->next_dva, size, IOMMU_READ | IOMMU_WRITE, s); - if (!ret) { - b->next_dva += size; + ret = ane_fw_alloc_at(b, ANE_FW_WINDOW_ANY, size, IOMMU_READ | IOMMU_WRITE, s); + if (!ret) dev_dbg(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); - } return ret; } @@ -1642,7 +1678,7 @@ static int ane_fw_prepare(struct ane_fw_backend *b) return ret; if (!(b->domain->pgsize_bitmap & SZ_16K) || !b->domain->ops->iotlb_sync_map || b->domain->geometry.aperture_start || - b->domain->geometry.aperture_end < ANE_FW_ALLOC_DVA + SZ_512M) + b->domain->geometry.aperture_end < ANE_FW_ALLOC_DVA + ANE_FW_WINDOW) return -ERANGE; b->memory = root_device_register("ane-fw-memory"); if (IS_ERR(b->memory)) @@ -1836,17 +1872,53 @@ static int ane_fw_startup(struct ane_fw_backend *b) static int ane_fw_program_map(void *ctx, size_t size, bool write, struct ane_fw_surface *out) { struct ane_fw_backend *b = ctx; - int ret; - if (!size || !IS_ALIGNED(size, SZ_16K) || - b->next_dva > ANE_FW_ALLOC_DVA + SZ_512M - size) + if (!size || !IS_ALIGNED(size, SZ_16K)) return -ENOSPC; - ret = ane_fw_alloc_mem(b, b->next_dva, size, - IOMMU_READ | (write ? IOMMU_WRITE : 0), true, out); - if (ret) - return ret; - b->next_dva += size; - return 0; + return ane_fw_alloc_mem(b, ANE_FW_WINDOW_ANY, size, + IOMMU_READ | (write ? IOMMU_WRITE : 0), true, out); +} + +/* + * A program surface the firmware no longer uses (ops->unmap). The session + * calls it with the engine up, so the DART invalidation that iommu_unmap() + * ends with reaches the engine before its pages and its DVA range go back. + * A short unmap keeps everything, as a failed mapping does. + */ +static void ane_fw_program_unmap(void *ctx, const struct ane_fw_surface *s) +{ + struct ane_fw_backend *b = ctx; + struct ane_fw_mapping *m, *found = NULL; + size_t unmapped; + + mutex_lock(&b->window_lock); + list_for_each_entry(m, &b->mappings, list) { + if (m->surface.dva == s->dva && m->surface.size == s->size && + m->surface.handle && drm_mm_node_allocated(&m->node)) { + found = m; + list_del(&m->list); + break; + } + } + mutex_unlock(&b->window_lock); + if (WARN_ON_ONCE(!found)) + return; + m = found; + unmapped = iommu_unmap(b->domain, s->dva, s->size); + if (unmapped != s->size) { + dev_err(&b->pdev->dev, "H17 ANE: program unmap short at %#llx: %zu of %zu; kept\n", + s->dva, unmapped, s->size); + mutex_lock(&b->window_lock); + list_add_tail(&m->list, &b->mappings); + mutex_unlock(&b->window_lock); + return; + } + dma_free_noncoherent(b->memory, s->size, m->surface.cpu, m->physical, + DMA_BIDIRECTIONAL); + mutex_lock(&b->window_lock); + drm_mm_remove_node(&m->node); + mutex_unlock(&b->window_lock); + kfree(m); } static void ane_fw_program_log(void *ctx, const char *item, u64 a, u64 c) @@ -1876,6 +1948,7 @@ static void ane_fw_program_busy(void *ctx, bool busy) static const struct ane_h16_session_ops ane_fw_session_ops = { .map = ane_fw_program_map, + .unmap = ane_fw_program_unmap, .log = ane_fw_program_log, .sync = ane_fw_program_sync, .busy = ane_fw_program_busy, @@ -2281,7 +2354,8 @@ int ane_fw_staged_init(struct platform_device *pdev, const struct ane_fw_platfor b->pdev = pdev; get_device(&pdev->dev); b->soc = soc; - b->next_dva = ANE_FW_ALLOC_DVA; + drm_mm_init(&b->dva_window, ANE_FW_ALLOC_DVA, ANE_FW_WINDOW); + mutex_init(&b->window_lock); mutex_init(&b->lock); INIT_LIST_HEAD(&b->mappings); if (of_property_count_strings(np, "reg-names") != soc->reg_count || From fd10fceaeacbf55e6765b7744f3be42cc5d09b0b Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Thu, 19 Nov 2026 11:04:37 +0100 Subject: [PATCH 4/5] accel/ane: h16: free private memory that never reached the IOMMU ane_h16_alloc_prot() puts a mapping on h->mappings before it allocates anything, so a 16K block, vmap() or sg table that fails leaves the mapping and the blocks it already got on the list. Before RUN the rollback frees them; from RUN on nothing does, and every program mapping that fails with -ENOMEM (a fragmented buddy allocator, a camera-sized surface) keeps its partial memory until reboot. Free the memory of a mapping that fails before iommu_map_sgtable() right away: nothing of it is visible to the engine. A mapping that iommu_map_sgtable() refused is still recorded and kept until reset, as before. The rollback frees mappings through the same helper. Assisted-by: LLM Signed-off-by: Bartosz Nitka --- drivers/accel/ane/ane_h16_backend.c | 42 ++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/drivers/accel/ane/ane_h16_backend.c b/drivers/accel/ane/ane_h16_backend.c index 4b1ca924c1759..9aee1320efa1f 100644 --- a/drivers/accel/ane/ane_h16_backend.c +++ b/drivers/accel/ane/ane_h16_backend.c @@ -1003,6 +1003,24 @@ static int ane_h16_map(void *ctx) return 0; } +/* Memory only: off h->mappings and not mapped. */ +static void ane_h16_mapping_free(struct ane_h16_mapping *m) +{ + unsigned int i, pages_per_block = SZ_16K / PAGE_SIZE; + + if (m->cpu) + vunmap(m->cpu); + for (i = 0; i < m->blocks; i++) + __free_pages(m->pages[i * pages_per_block], get_order(SZ_16K)); + kvfree(m->pages); + kfree(m); +} + +/* + * Map size bytes of fresh zeroed memory at dva. Memory that never reached + * the IOMMU is freed on failure; a failed or partial iommu_map_sgtable() + * keeps the mapping and its memory until reset. + */ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, int prot, struct ane_fw_surface *out) { @@ -1025,15 +1043,15 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in return -ENOMEM; m->size = size; m->dva = dva; - list_add_tail(&m->node, &h->mappings); + ret = -ENOMEM; m->pages = kvcalloc(size / PAGE_SIZE, sizeof(*m->pages), GFP_KERNEL); if (!m->pages) - return -ENOMEM; + goto free; for (i = 0; i < size / SZ_16K; i++) { struct page *page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); if (!page) - return -ENOMEM; + goto free; for (j = 0; j < pages_per_block; j++) m->pages[i * pages_per_block + j] = page + j; m->blocks++; @@ -1041,10 +1059,12 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in /* The coherent-client gate permits matching cached CPU/device aliases. */ m->cpu = vmap(m->pages, size / PAGE_SIZE, VM_MAP, PAGE_KERNEL); if (!m->cpu) - return -ENOMEM; + goto free; ret = sg_alloc_table_from_pages(&sgt, m->pages, size / PAGE_SIZE, 0, size, GFP_KERNEL); if (ret) - return ret; + goto free; + /* Recorded before publication, including a failed/partial mapping. */ + list_add_tail(&m->node, &h->mappings); /* A failed call has already unmapped what it mapped. */ mapped = iommu_map_sgtable(h->domain, dva, &sgt, prot); sg_free_table(&sgt); @@ -1053,6 +1073,9 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in m->mapped = mapped; *out = (struct ane_fw_surface) { .cpu = m->cpu, .dva = dva, .size = size }; return 0; +free: + ane_h16_mapping_free(m); + return ret; } static int ane_h16_alloc_at(void *ctx, u64 dva, size_t size, struct ane_fw_surface *out) @@ -1465,20 +1488,15 @@ static const struct ane_fw_transport_ops ane_h16_transport_ops = { static void ane_h16_release(struct ane_h16_device *h) { struct ane_h16_mapping *m, *next; - unsigned int i, pages_per_block = SZ_16K / PAGE_SIZE; + unsigned int i; if (h->boot.ran || h->pinned || !h->acquired) return; list_for_each_entry_safe_reverse(m, next, &h->mappings, node) { if (m->mapped) iommu_unmap(h->domain, m->dva, m->mapped); - if (m->cpu) - vunmap(m->cpu); - for (i = 0; i < m->blocks; i++) - __free_pages(m->pages[i * pages_per_block], get_order(SZ_16K)); - kvfree(m->pages); list_del(&m->node); - kfree(m); + ane_h16_mapping_free(m); } if (h->attached) iommu_detach_device(h->domain, h->dev); From 049ae047f34fab95188c24a32457ccd0d548a07a Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Thu, 19 Nov 2026 11:04:51 +0100 Subject: [PATCH 5/5] accel/ane: h16: return program DVA ranges and memory The H16 backend hands out program mappings by bumping a pointer through a 1 GiB DVA window and never gives anything back: once the window is used up, every program mapping fails with -ENOSPC until reboot. A client that loads programs of many shapes gets there, as the H17 backend did on the J700 after 337 loads of a held-out layer sweep. Allocate program ranges from a drm_mm instead, and implement the session's ops->unmap: find the program mapping, unmap it from the DARTs (iommu_unmap() invalidates their TLBs before it returns), then free its pages and its range. A short unmap keeps the mapping, its memory and its range, as a failed mapping does. SHAREDMALLOC, preload, heap tail and identity MMIO mappings have no range in the window and are never unmapped. The session calls ops->unmap only with the engine up, never while the session is asleep for system sleep or a recovery power cycle, so the invalidation reaches a powered engine. A new lock guards the mappings list and the window, since SHAREDMALLOC can be mapped while a program is unmapped. Assisted-by: LLM Signed-off-by: Bartosz Nitka --- drivers/accel/ane/ane_h16_backend.c | 111 +++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 17 deletions(-) diff --git a/drivers/accel/ane/ane_h16_backend.c b/drivers/accel/ane/ane_h16_backend.c index 9aee1320efa1f..534cd0e8f193a 100644 --- a/drivers/accel/ane/ane_h16_backend.c +++ b/drivers/accel/ane/ane_h16_backend.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Staged H16 backend, using the accepted H16 cold-boot facts, round 5. */ +#include #include #include #include @@ -36,6 +37,8 @@ #define ANE_H16_TX_WORDS 272 /* Private program window above the SHAREDMALLOC range; host policy. */ #define ANE_H16_PROGRAM_WINDOW SZ_1G +/* ane_h16_alloc_prot(): a DVA range from the program window. */ +#define ANE_H16_PROGRAM_ANY U64_MAX struct ane_h16_mapping { struct list_head node; @@ -45,6 +48,8 @@ struct ane_h16_mapping { struct page **pages; u32 blocks; void *cpu; + /* The DVA range, for a mapping in the program window. */ + struct drm_mm_node range; }; struct ane_h16_device { @@ -61,6 +66,12 @@ struct ane_h16_device { u32 powered; u32 darts_powered; u64 preload_phys[2]; + /* + * Guards mappings and program_window. Program mappings come and go + * under the session lock while the engine is up; SHAREDMALLOC may be + * mapped at the same time. + */ + struct mutex window_lock; struct list_head mappings; struct iommu_domain *domain; struct iommu_group *group; @@ -93,7 +104,9 @@ struct ane_h16_device { u32 tx_write; bool rtk_failed; u64 shared_next; - u64 program_next; + /* Program sections and surfaces: [program_start, program_end). */ + struct drm_mm program_window; + u64 program_start; u64 program_end; struct dentry *debugfs; bool acquired; @@ -885,7 +898,9 @@ static int ane_h16_map_range(struct ane_h16_device *h, u64 dva, u64 pa, return -ENOMEM; m->dva = dva; m->size = size; + mutex_lock(&h->window_lock); list_add_tail(&m->node, &h->mappings); + mutex_unlock(&h->window_lock); ret = iommu_map(h->domain, dva, pa, size, prot, GFP_KERNEL); if (!ret) m->mapped = size; @@ -1003,7 +1018,7 @@ static int ane_h16_map(void *ctx) return 0; } -/* Memory only: off h->mappings and not mapped. */ +/* Memory only: off h->mappings, its range (if any) already given back. */ static void ane_h16_mapping_free(struct ane_h16_mapping *m) { unsigned int i, pages_per_block = SZ_16K / PAGE_SIZE; @@ -1017,9 +1032,10 @@ static void ane_h16_mapping_free(struct ane_h16_mapping *m) } /* - * Map size bytes of fresh zeroed memory at dva. Memory that never reached - * the IOMMU is freed on failure; a failed or partial iommu_map_sgtable() - * keeps the mapping and its memory until reset. + * Map size bytes of fresh zeroed memory at dva, or with ANE_H16_PROGRAM_ANY + * at a range of the program window. Memory that never reached the IOMMU is + * freed on failure; a failed or partial iommu_map_sgtable() keeps the + * mapping, its memory and its range until reset. */ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, int prot, struct ane_fw_surface *out) @@ -1035,14 +1051,13 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in dev_err(h->dev, "H16 private surfaces require a coherent client\n"); return -EOPNOTSUPP; } - if (!size || !IS_ALIGNED(size, SZ_16K) || !IS_ALIGNED(dva, SZ_16K) || - PAGE_SIZE > SZ_16K) + if (!size || !IS_ALIGNED(size, SZ_16K) || PAGE_SIZE > SZ_16K || + (dva != ANE_H16_PROGRAM_ANY && !IS_ALIGNED(dva, SZ_16K))) return -EINVAL; m = kzalloc_obj(*m); if (!m) return -ENOMEM; m->size = size; - m->dva = dva; ret = -ENOMEM; m->pages = kvcalloc(size / PAGE_SIZE, sizeof(*m->pages), GFP_KERNEL); if (!m->pages) @@ -1063,8 +1078,22 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in ret = sg_alloc_table_from_pages(&sgt, m->pages, size / PAGE_SIZE, 0, size, GFP_KERNEL); if (ret) goto free; + mutex_lock(&h->window_lock); + if (dva == ANE_H16_PROGRAM_ANY) { + ret = drm_mm_insert_node_in_range(&h->program_window, &m->range, size, + SZ_16K, 0, 0, ANE_H16_PROGRAM_WINDOW, + DRM_MM_INSERT_LOW); + if (ret) { + mutex_unlock(&h->window_lock); + sg_free_table(&sgt); + goto free; + } + dva = h->program_start + m->range.start; + } + m->dva = dva; /* Recorded before publication, including a failed/partial mapping. */ list_add_tail(&m->node, &h->mappings); + mutex_unlock(&h->window_lock); /* A failed call has already unmapped what it mapped. */ mapped = iommu_map_sgtable(h->domain, dva, &sgt, prot); sg_free_table(&sgt); @@ -1087,21 +1116,61 @@ static int ane_h16_alloc_at(void *ctx, u64 dva, size_t size, struct ane_fw_surfa static int ane_h16_program_map(void *ctx, size_t size, bool write, struct ane_fw_surface *out) { struct ane_h16_device *h = ctx; - u64 dva = h->program_next; int ret; - if (!size || size > h->program_end - dva) + if (!size || size > ANE_H16_PROGRAM_WINDOW) return -ENOSPC; - /* A failed mapping keeps its range; DVAs are never reused before reset. */ - h->program_next += size; - ret = ane_h16_alloc_prot(h, dva, size, + ret = ane_h16_alloc_prot(h, ANE_H16_PROGRAM_ANY, size, IOMMU_READ | IOMMU_CACHE | (write ? IOMMU_WRITE : 0), out); if (!ret) - dev_info(h->dev, "H16 program mapping %llx bytes %zx%s\n", dva, size, + dev_info(h->dev, "H16 program mapping %llx bytes %zx%s\n", out->dva, size, write ? " writable" : ""); return ret; } +/* + * A program surface the firmware no longer uses (ops->unmap). The session + * calls it with the engine up and its lock held. iommu_unmap() ends with + * the DART TLB invalidation, so the engine cannot reach the memory once it + * returns; only then do the pages and the DVA range go back. A short unmap + * keeps everything, as a failed mapping does. Only program-window mappings + * qualify: SHAREDMALLOC, preload, heap tail and MMIO stay until reset. + */ +static void ane_h16_program_unmap(void *ctx, const struct ane_fw_surface *s) +{ + struct ane_h16_device *h = ctx; + struct ane_h16_mapping *m, *found = NULL; + size_t unmapped; + + mutex_lock(&h->window_lock); + list_for_each_entry(m, &h->mappings, node) { + if (m->dva == s->dva && m->size == s->size && m->mapped == m->size && + drm_mm_node_allocated(&m->range)) { + found = m; + list_del(&m->node); + break; + } + } + mutex_unlock(&h->window_lock); + if (WARN_ON_ONCE(!found)) + return; + m = found; + unmapped = iommu_unmap(h->domain, m->dva, m->size); + if (unmapped != m->size) { + dev_err(h->dev, "H16 program unmap short at %llx: %zx of %zx; kept\n", + m->dva, unmapped, m->size); + mutex_lock(&h->window_lock); + list_add_tail(&m->node, &h->mappings); + mutex_unlock(&h->window_lock); + return; + } + dev_dbg(h->dev, "H16 program unmapped %llx bytes %zx\n", m->dva, m->size); + mutex_lock(&h->window_lock); + drm_mm_remove_node(&m->range); + mutex_unlock(&h->window_lock); + ane_h16_mapping_free(m); +} + static u64 ane_h16_load64(void *ctx, const void *p) { return le64_to_cpu(READ_ONCE(*(const __le64 *)p)); @@ -1462,6 +1531,7 @@ static void ane_h16_hung(void *ctx, int error) static const struct ane_h16_session_ops ane_h16_session_ops = { .map = ane_h16_program_map, + .unmap = ane_h16_program_unmap, .log = ane_h16_log, .busy = ane_h16_busy, .hung = ane_h16_hung, @@ -1496,6 +1566,8 @@ static void ane_h16_release(struct ane_h16_device *h) if (m->mapped) iommu_unmap(h->domain, m->dva, m->mapped); list_del(&m->node); + if (drm_mm_node_allocated(&m->range)) + drm_mm_remove_node(&m->range); ane_h16_mapping_free(m); } if (h->attached) @@ -1806,8 +1878,8 @@ static int ane_h16_preload_identity(struct ane_h16_device *h) b->max_restarts = ANE_H16_STARTUP_RESTARTS; } h->shared_next = b->build->shared_dva; - h->program_next = b->build->shared_dva + ANE_FW_SHARED_TOTAL; - h->program_end = h->program_next + ANE_H16_PROGRAM_WINDOW; + h->program_start = b->build->shared_dva + ANE_FW_SHARED_TOTAL; + h->program_end = h->program_start + ANE_H16_PROGRAM_WINDOW; dev_info(h->dev, "H16 preload identity %s accepted\n", identity); out: vfree(second); @@ -1845,7 +1917,7 @@ static int ane_h16_program_stage(struct ane_h16_device *h) } h->program_ready = true; ane_h16_irq_wait_set(h, true); - dev_info(h->dev, "H16 stage 5 program window %llx..%llx%s\n", h->program_next, + dev_info(h->dev, "H16 stage 5 program window %llx..%llx%s\n", h->program_start, h->program_end - 1, h->clpc ? ", CLPC input attached" : ""); return 0; } @@ -2074,7 +2146,10 @@ int ane_h16_backend_probe(struct platform_device *pdev, const struct ane_fw_plat h->irq = -1; spin_lock_init(&h->irq_lock); init_waitqueue_head(&h->irq_wq); + mutex_init(&h->window_lock); INIT_LIST_HEAD(&h->mappings); + /* Offsets from program_start, which the preload's build decides. */ + drm_mm_init(&h->program_window, 0, ANE_H16_PROGRAM_WINDOW); h->boot.soc = soc; /* The build, and with it the DVA layout, is chosen from the preload. */ h->boot.build = NULL; @@ -2115,6 +2190,7 @@ int ane_h16_backend_probe(struct platform_device *pdev, const struct ane_fw_plat goto unlock; free: dev_err(h->dev, "H16 DT description refused, error %d\n", ret); + drm_mm_takedown(&h->program_window); kfree(h); unlock: mutex_unlock(&ane_h16_lock); @@ -2146,6 +2222,7 @@ void ane_h16_backend_remove(struct platform_device *pdev) dev_warn(h->dev, "H16 remove held; reboot reclaims firmware DMA\n"); } else { ane_h16_release(h); + drm_mm_takedown(&h->program_window); kfree(h); } mutex_unlock(&ane_h16_lock);