diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c index 2dc8d2239b89c..6aa6f1a29ad9a 100644 --- a/drivers/accel/ane/ane_fw_cold.c +++ b/drivers/accel/ane/ane_fw_cold.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Shared firmware backend. H17 is the first measured cold power provider. */ #include +#include #include #include #include @@ -31,6 +32,10 @@ #define ANE_FW_IPC_DVA 0x100003d0000ULL #define ANE_FW_HEAP_DVA 0x100003e4000ULL #define ANE_FW_ALLOC_DVA 0x10001300000ULL +/* SHAREDMALLOC and program mappings: DVAs from ANE_FW_ALLOC_DVA on. */ +#define ANE_FW_WINDOW SZ_512M +/* ane_fw_alloc_mem(): a DVA range from the window. */ +#define ANE_FW_WINDOW_ANY U64_MAX #define ANE_FW_TIMEOUT_NS (3ULL * NSEC_PER_SEC) enum ane_fw_stage { @@ -47,6 +52,8 @@ struct ane_fw_mapping { struct list_head list; struct ane_fw_surface surface; dma_addr_t physical; + /* The DVA range, for a mapping in the window. */ + struct drm_mm_node node; }; struct ane_fw_backend { @@ -113,7 +120,13 @@ struct ane_fw_backend { u64 sys_off_at, sys_off_ns; struct ane_fw_cold_state cold; u64 ranges[ANE_FW_DAPF_MAX * 2]; - u64 next_dva; + /* + * The DVA window and the mappings list. Program mappings come and go + * under the session lock while the engine is up; the others are made + * with the session closed. + */ + struct drm_mm dva_window; + struct mutex window_lock; enum ane_fw_stage stage; bool retained; /* Brings the engine up to the program stage after a loader hand-off. */ @@ -1282,6 +1295,10 @@ static int ane_fw_alloc_mem(struct ane_fw_backend *b, u64 dva, size_t size, struct ane_fw_mapping *m; int ret; + /* No memory for a range the window cannot hold. */ + if (dva == ANE_FW_WINDOW_ANY && size > ANE_FW_WINDOW) + return -ENOSPC; + m = kzalloc_obj(*m); if (!m) return -ENOMEM; @@ -1301,10 +1318,32 @@ static int ane_fw_alloc_mem(struct ane_fw_backend *b, u64 dva, size_t size, dma_sync_single_for_device(b->memory, m->physical, size, DMA_BIDIRECTIONAL); m->surface.handle = m->physical; } + mutex_lock(&b->window_lock); + if (dva == ANE_FW_WINDOW_ANY) { + ret = drm_mm_insert_node_in_range(&b->dva_window, &m->node, size, SZ_16K, 0, + ANE_FW_ALLOC_DVA, + ANE_FW_ALLOC_DVA + ANE_FW_WINDOW, + DRM_MM_INSERT_LOW); + if (ret) { + mutex_unlock(&b->window_lock); + if (cached) + dma_free_noncoherent(b->memory, size, m->surface.cpu, + m->physical, DMA_BIDIRECTIONAL); + else + dma_free_coherent(b->memory, size, m->surface.cpu, m->physical); + kfree(m); + return ret; + } + dva = m->node.start; + } m->surface.dva = dva; m->surface.size = size; - /* Record before publication, including a failed/partial mapping. */ + /* + * Record before publication, including a failed/partial mapping: it + * keeps its memory and range until reset. + */ list_add_tail(&m->list, &b->mappings); + mutex_unlock(&b->window_lock); ret = ane_fw_map(b, dva, m->physical, size, prot); if (!ret) *s = m->surface; @@ -1322,14 +1361,11 @@ static int ane_fw_alloc(void *ctx, size_t size, u32 tag, struct ane_fw_surface * struct ane_fw_backend *b = ctx; int ret; - if (!size || size > ANE_FW_SHARED_TOTAL || - b->next_dva > ANE_FW_ALLOC_DVA + SZ_512M - size) + if (!size || size > ANE_FW_SHARED_TOTAL) return -ENOSPC; - ret = ane_fw_alloc_at(b, b->next_dva, size, IOMMU_READ | IOMMU_WRITE, s); - if (!ret) { - b->next_dva += size; + ret = ane_fw_alloc_at(b, ANE_FW_WINDOW_ANY, size, IOMMU_READ | IOMMU_WRITE, s); + if (!ret) dev_dbg(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); - } return ret; } @@ -1642,7 +1678,7 @@ static int ane_fw_prepare(struct ane_fw_backend *b) return ret; if (!(b->domain->pgsize_bitmap & SZ_16K) || !b->domain->ops->iotlb_sync_map || b->domain->geometry.aperture_start || - b->domain->geometry.aperture_end < ANE_FW_ALLOC_DVA + SZ_512M) + b->domain->geometry.aperture_end < ANE_FW_ALLOC_DVA + ANE_FW_WINDOW) return -ERANGE; b->memory = root_device_register("ane-fw-memory"); if (IS_ERR(b->memory)) @@ -1836,17 +1872,53 @@ static int ane_fw_startup(struct ane_fw_backend *b) static int ane_fw_program_map(void *ctx, size_t size, bool write, struct ane_fw_surface *out) { struct ane_fw_backend *b = ctx; - int ret; - if (!size || !IS_ALIGNED(size, SZ_16K) || - b->next_dva > ANE_FW_ALLOC_DVA + SZ_512M - size) + if (!size || !IS_ALIGNED(size, SZ_16K)) return -ENOSPC; - ret = ane_fw_alloc_mem(b, b->next_dva, size, - IOMMU_READ | (write ? IOMMU_WRITE : 0), true, out); - if (ret) - return ret; - b->next_dva += size; - return 0; + return ane_fw_alloc_mem(b, ANE_FW_WINDOW_ANY, size, + IOMMU_READ | (write ? IOMMU_WRITE : 0), true, out); +} + +/* + * A program surface the firmware no longer uses (ops->unmap). The session + * calls it with the engine up, so the DART invalidation that iommu_unmap() + * ends with reaches the engine before its pages and its DVA range go back. + * A short unmap keeps everything, as a failed mapping does. + */ +static void ane_fw_program_unmap(void *ctx, const struct ane_fw_surface *s) +{ + struct ane_fw_backend *b = ctx; + struct ane_fw_mapping *m, *found = NULL; + size_t unmapped; + + mutex_lock(&b->window_lock); + list_for_each_entry(m, &b->mappings, list) { + if (m->surface.dva == s->dva && m->surface.size == s->size && + m->surface.handle && drm_mm_node_allocated(&m->node)) { + found = m; + list_del(&m->list); + break; + } + } + mutex_unlock(&b->window_lock); + if (WARN_ON_ONCE(!found)) + return; + m = found; + unmapped = iommu_unmap(b->domain, s->dva, s->size); + if (unmapped != s->size) { + dev_err(&b->pdev->dev, "H17 ANE: program unmap short at %#llx: %zu of %zu; kept\n", + s->dva, unmapped, s->size); + mutex_lock(&b->window_lock); + list_add_tail(&m->list, &b->mappings); + mutex_unlock(&b->window_lock); + return; + } + dma_free_noncoherent(b->memory, s->size, m->surface.cpu, m->physical, + DMA_BIDIRECTIONAL); + mutex_lock(&b->window_lock); + drm_mm_remove_node(&m->node); + mutex_unlock(&b->window_lock); + kfree(m); } static void ane_fw_program_log(void *ctx, const char *item, u64 a, u64 c) @@ -1876,6 +1948,7 @@ static void ane_fw_program_busy(void *ctx, bool busy) static const struct ane_h16_session_ops ane_fw_session_ops = { .map = ane_fw_program_map, + .unmap = ane_fw_program_unmap, .log = ane_fw_program_log, .sync = ane_fw_program_sync, .busy = ane_fw_program_busy, @@ -2281,7 +2354,8 @@ int ane_fw_staged_init(struct platform_device *pdev, const struct ane_fw_platfor b->pdev = pdev; get_device(&pdev->dev); b->soc = soc; - b->next_dva = ANE_FW_ALLOC_DVA; + drm_mm_init(&b->dva_window, ANE_FW_ALLOC_DVA, ANE_FW_WINDOW); + mutex_init(&b->window_lock); mutex_init(&b->lock); INIT_LIST_HEAD(&b->mappings); if (of_property_count_strings(np, "reg-names") != soc->reg_count || diff --git a/drivers/accel/ane/ane_h16_backend.c b/drivers/accel/ane/ane_h16_backend.c index 4b1ca924c1759..534cd0e8f193a 100644 --- a/drivers/accel/ane/ane_h16_backend.c +++ b/drivers/accel/ane/ane_h16_backend.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Staged H16 backend, using the accepted H16 cold-boot facts, round 5. */ +#include #include #include #include @@ -36,6 +37,8 @@ #define ANE_H16_TX_WORDS 272 /* Private program window above the SHAREDMALLOC range; host policy. */ #define ANE_H16_PROGRAM_WINDOW SZ_1G +/* ane_h16_alloc_prot(): a DVA range from the program window. */ +#define ANE_H16_PROGRAM_ANY U64_MAX struct ane_h16_mapping { struct list_head node; @@ -45,6 +48,8 @@ struct ane_h16_mapping { struct page **pages; u32 blocks; void *cpu; + /* The DVA range, for a mapping in the program window. */ + struct drm_mm_node range; }; struct ane_h16_device { @@ -61,6 +66,12 @@ struct ane_h16_device { u32 powered; u32 darts_powered; u64 preload_phys[2]; + /* + * Guards mappings and program_window. Program mappings come and go + * under the session lock while the engine is up; SHAREDMALLOC may be + * mapped at the same time. + */ + struct mutex window_lock; struct list_head mappings; struct iommu_domain *domain; struct iommu_group *group; @@ -93,7 +104,9 @@ struct ane_h16_device { u32 tx_write; bool rtk_failed; u64 shared_next; - u64 program_next; + /* Program sections and surfaces: [program_start, program_end). */ + struct drm_mm program_window; + u64 program_start; u64 program_end; struct dentry *debugfs; bool acquired; @@ -885,7 +898,9 @@ static int ane_h16_map_range(struct ane_h16_device *h, u64 dva, u64 pa, return -ENOMEM; m->dva = dva; m->size = size; + mutex_lock(&h->window_lock); list_add_tail(&m->node, &h->mappings); + mutex_unlock(&h->window_lock); ret = iommu_map(h->domain, dva, pa, size, prot, GFP_KERNEL); if (!ret) m->mapped = size; @@ -1003,6 +1018,25 @@ static int ane_h16_map(void *ctx) return 0; } +/* Memory only: off h->mappings, its range (if any) already given back. */ +static void ane_h16_mapping_free(struct ane_h16_mapping *m) +{ + unsigned int i, pages_per_block = SZ_16K / PAGE_SIZE; + + if (m->cpu) + vunmap(m->cpu); + for (i = 0; i < m->blocks; i++) + __free_pages(m->pages[i * pages_per_block], get_order(SZ_16K)); + kvfree(m->pages); + kfree(m); +} + +/* + * Map size bytes of fresh zeroed memory at dva, or with ANE_H16_PROGRAM_ANY + * at a range of the program window. Memory that never reached the IOMMU is + * freed on failure; a failed or partial iommu_map_sgtable() keeps the + * mapping, its memory and its range until reset. + */ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, int prot, struct ane_fw_surface *out) { @@ -1017,23 +1051,22 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in dev_err(h->dev, "H16 private surfaces require a coherent client\n"); return -EOPNOTSUPP; } - if (!size || !IS_ALIGNED(size, SZ_16K) || !IS_ALIGNED(dva, SZ_16K) || - PAGE_SIZE > SZ_16K) + if (!size || !IS_ALIGNED(size, SZ_16K) || PAGE_SIZE > SZ_16K || + (dva != ANE_H16_PROGRAM_ANY && !IS_ALIGNED(dva, SZ_16K))) return -EINVAL; m = kzalloc_obj(*m); if (!m) return -ENOMEM; m->size = size; - m->dva = dva; - list_add_tail(&m->node, &h->mappings); + ret = -ENOMEM; m->pages = kvcalloc(size / PAGE_SIZE, sizeof(*m->pages), GFP_KERNEL); if (!m->pages) - return -ENOMEM; + goto free; for (i = 0; i < size / SZ_16K; i++) { struct page *page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); if (!page) - return -ENOMEM; + goto free; for (j = 0; j < pages_per_block; j++) m->pages[i * pages_per_block + j] = page + j; m->blocks++; @@ -1041,10 +1074,26 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in /* The coherent-client gate permits matching cached CPU/device aliases. */ m->cpu = vmap(m->pages, size / PAGE_SIZE, VM_MAP, PAGE_KERNEL); if (!m->cpu) - return -ENOMEM; + goto free; ret = sg_alloc_table_from_pages(&sgt, m->pages, size / PAGE_SIZE, 0, size, GFP_KERNEL); if (ret) - return ret; + goto free; + mutex_lock(&h->window_lock); + if (dva == ANE_H16_PROGRAM_ANY) { + ret = drm_mm_insert_node_in_range(&h->program_window, &m->range, size, + SZ_16K, 0, 0, ANE_H16_PROGRAM_WINDOW, + DRM_MM_INSERT_LOW); + if (ret) { + mutex_unlock(&h->window_lock); + sg_free_table(&sgt); + goto free; + } + dva = h->program_start + m->range.start; + } + m->dva = dva; + /* Recorded before publication, including a failed/partial mapping. */ + list_add_tail(&m->node, &h->mappings); + mutex_unlock(&h->window_lock); /* A failed call has already unmapped what it mapped. */ mapped = iommu_map_sgtable(h->domain, dva, &sgt, prot); sg_free_table(&sgt); @@ -1053,6 +1102,9 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in m->mapped = mapped; *out = (struct ane_fw_surface) { .cpu = m->cpu, .dva = dva, .size = size }; return 0; +free: + ane_h16_mapping_free(m); + return ret; } static int ane_h16_alloc_at(void *ctx, u64 dva, size_t size, struct ane_fw_surface *out) @@ -1064,21 +1116,61 @@ static int ane_h16_alloc_at(void *ctx, u64 dva, size_t size, struct ane_fw_surfa static int ane_h16_program_map(void *ctx, size_t size, bool write, struct ane_fw_surface *out) { struct ane_h16_device *h = ctx; - u64 dva = h->program_next; int ret; - if (!size || size > h->program_end - dva) + if (!size || size > ANE_H16_PROGRAM_WINDOW) return -ENOSPC; - /* A failed mapping keeps its range; DVAs are never reused before reset. */ - h->program_next += size; - ret = ane_h16_alloc_prot(h, dva, size, + ret = ane_h16_alloc_prot(h, ANE_H16_PROGRAM_ANY, size, IOMMU_READ | IOMMU_CACHE | (write ? IOMMU_WRITE : 0), out); if (!ret) - dev_info(h->dev, "H16 program mapping %llx bytes %zx%s\n", dva, size, + dev_info(h->dev, "H16 program mapping %llx bytes %zx%s\n", out->dva, size, write ? " writable" : ""); return ret; } +/* + * A program surface the firmware no longer uses (ops->unmap). The session + * calls it with the engine up and its lock held. iommu_unmap() ends with + * the DART TLB invalidation, so the engine cannot reach the memory once it + * returns; only then do the pages and the DVA range go back. A short unmap + * keeps everything, as a failed mapping does. Only program-window mappings + * qualify: SHAREDMALLOC, preload, heap tail and MMIO stay until reset. + */ +static void ane_h16_program_unmap(void *ctx, const struct ane_fw_surface *s) +{ + struct ane_h16_device *h = ctx; + struct ane_h16_mapping *m, *found = NULL; + size_t unmapped; + + mutex_lock(&h->window_lock); + list_for_each_entry(m, &h->mappings, node) { + if (m->dva == s->dva && m->size == s->size && m->mapped == m->size && + drm_mm_node_allocated(&m->range)) { + found = m; + list_del(&m->node); + break; + } + } + mutex_unlock(&h->window_lock); + if (WARN_ON_ONCE(!found)) + return; + m = found; + unmapped = iommu_unmap(h->domain, m->dva, m->size); + if (unmapped != m->size) { + dev_err(h->dev, "H16 program unmap short at %llx: %zx of %zx; kept\n", + m->dva, unmapped, m->size); + mutex_lock(&h->window_lock); + list_add_tail(&m->node, &h->mappings); + mutex_unlock(&h->window_lock); + return; + } + dev_dbg(h->dev, "H16 program unmapped %llx bytes %zx\n", m->dva, m->size); + mutex_lock(&h->window_lock); + drm_mm_remove_node(&m->range); + mutex_unlock(&h->window_lock); + ane_h16_mapping_free(m); +} + static u64 ane_h16_load64(void *ctx, const void *p) { return le64_to_cpu(READ_ONCE(*(const __le64 *)p)); @@ -1439,6 +1531,7 @@ static void ane_h16_hung(void *ctx, int error) static const struct ane_h16_session_ops ane_h16_session_ops = { .map = ane_h16_program_map, + .unmap = ane_h16_program_unmap, .log = ane_h16_log, .busy = ane_h16_busy, .hung = ane_h16_hung, @@ -1465,20 +1558,17 @@ static const struct ane_fw_transport_ops ane_h16_transport_ops = { static void ane_h16_release(struct ane_h16_device *h) { struct ane_h16_mapping *m, *next; - unsigned int i, pages_per_block = SZ_16K / PAGE_SIZE; + unsigned int i; if (h->boot.ran || h->pinned || !h->acquired) return; list_for_each_entry_safe_reverse(m, next, &h->mappings, node) { if (m->mapped) iommu_unmap(h->domain, m->dva, m->mapped); - if (m->cpu) - vunmap(m->cpu); - for (i = 0; i < m->blocks; i++) - __free_pages(m->pages[i * pages_per_block], get_order(SZ_16K)); - kvfree(m->pages); list_del(&m->node); - kfree(m); + if (drm_mm_node_allocated(&m->range)) + drm_mm_remove_node(&m->range); + ane_h16_mapping_free(m); } if (h->attached) iommu_detach_device(h->domain, h->dev); @@ -1788,8 +1878,8 @@ static int ane_h16_preload_identity(struct ane_h16_device *h) b->max_restarts = ANE_H16_STARTUP_RESTARTS; } h->shared_next = b->build->shared_dva; - h->program_next = b->build->shared_dva + ANE_FW_SHARED_TOTAL; - h->program_end = h->program_next + ANE_H16_PROGRAM_WINDOW; + h->program_start = b->build->shared_dva + ANE_FW_SHARED_TOTAL; + h->program_end = h->program_start + ANE_H16_PROGRAM_WINDOW; dev_info(h->dev, "H16 preload identity %s accepted\n", identity); out: vfree(second); @@ -1827,7 +1917,7 @@ static int ane_h16_program_stage(struct ane_h16_device *h) } h->program_ready = true; ane_h16_irq_wait_set(h, true); - dev_info(h->dev, "H16 stage 5 program window %llx..%llx%s\n", h->program_next, + dev_info(h->dev, "H16 stage 5 program window %llx..%llx%s\n", h->program_start, h->program_end - 1, h->clpc ? ", CLPC input attached" : ""); return 0; } @@ -2056,7 +2146,10 @@ int ane_h16_backend_probe(struct platform_device *pdev, const struct ane_fw_plat h->irq = -1; spin_lock_init(&h->irq_lock); init_waitqueue_head(&h->irq_wq); + mutex_init(&h->window_lock); INIT_LIST_HEAD(&h->mappings); + /* Offsets from program_start, which the preload's build decides. */ + drm_mm_init(&h->program_window, 0, ANE_H16_PROGRAM_WINDOW); h->boot.soc = soc; /* The build, and with it the DVA layout, is chosen from the preload. */ h->boot.build = NULL; @@ -2097,6 +2190,7 @@ int ane_h16_backend_probe(struct platform_device *pdev, const struct ane_fw_plat goto unlock; free: dev_err(h->dev, "H16 DT description refused, error %d\n", ret); + drm_mm_takedown(&h->program_window); kfree(h); unlock: mutex_unlock(&ane_h16_lock); @@ -2128,6 +2222,7 @@ void ane_h16_backend_remove(struct platform_device *pdev) dev_warn(h->dev, "H16 remove held; reboot reclaims firmware DMA\n"); } else { ane_h16_release(h); + drm_mm_takedown(&h->program_window); kfree(h); } mutex_unlock(&ane_h16_lock); diff --git a/drivers/accel/ane/ane_h16_program.c b/drivers/accel/ane/ane_h16_program.c index 9132cc2ddaded..d6a72ef64ec79 100644 --- a/drivers/accel/ane/ane_h16_program.c +++ b/drivers/accel/ane/ane_h16_program.c @@ -597,8 +597,9 @@ static int h16_send(struct ane_h16_session *s, unsigned int ch, const void *comm /* * The transport copies commands into a pool arena. LOAD and CREATE stay in - * it; each EXECUTE sequence gives its space back once the firmware reported - * the request complete. Refuse a sequence up front when it cannot fit, so + * it while any program of this firmware boot is alive (h16_pool_rewind()); + * each EXECUTE sequence gives its space back once the firmware reported the + * request complete. Refuse a sequence up front when it cannot fit, so * running out of arena is a clean -ENOSPC and never a half-sent sequence. */ static int h16_pool_room(struct ane_h16_session *s, const size_t *lengths, @@ -617,6 +618,23 @@ static int h16_pool_room(struct ane_h16_session *s, const size_t *lengths, return 0; } +/* + * The last program of this firmware boot was torn down cleanly and nothing + * is in flight: the firmware is done with every command published since the + * LOAD that found no program alive (LOAD, CREATE, gates, TERMINATE and + * UNLOAD), so give their arena back. Without this the arena filled after + * about two hundred programs and every LOAD failed with -ENOSPC until the + * firmware was booted again. + */ +static void h16_pool_rewind(struct ane_h16_session *s) +{ + if (!s->pool_base_set || !list_empty(&s->programs) || !list_empty(&s->held) || + s->running || s->active || s->quarantined) + return; + if (!ane_fw_transport_release(s->transport, s->pool_base)) + s->pool_base_set = false; +} + static void h16_sync(struct ane_h16_session *s, const struct ane_fw_surface *surface, bool for_device) { @@ -640,6 +658,37 @@ void ane_h16_session_sync(struct ane_h16_session *s, struct ane_h16_program *p, } EXPORT_SYMBOL_GPL(ane_h16_session_sync); +/* + * Spare memory can go back to the backend: it has ops->unmap and the engine + * is up, so the DART invalidation that comes with the unmap reaches it. + */ +static bool h16_can_unmap(const struct ane_h16_session *s) +{ + return s->ops->unmap && !s->asleep && !s->parked; +} + +static struct ane_fw_surface h16_spare_remove(struct ane_h16_session *s, unsigned int i) +{ + struct ane_fw_surface surface = s->spares[i].surface; + + s->spare_bytes -= surface.size; + s->spare_count--; + memmove(&s->spares[i], &s->spares[i + 1], (s->spare_count - i) * sizeof(s->spares[0])); + return surface; +} + +/* Free the oldest spares until at most @count entries and @bytes remain. */ +static void h16_spares_trim(struct ane_h16_session *s, unsigned int count, size_t bytes) +{ + if (!h16_can_unmap(s)) + return; + while (s->spare_count && (s->spare_count > count || s->spare_bytes > bytes)) { + struct ane_fw_surface surface = h16_spare_remove(s, 0); + + s->ops->unmap(s->ctx, &surface); + } +} + static bool h16_take_spare(struct ane_h16_session *s, size_t size, bool write, struct ane_fw_surface *out) { @@ -648,8 +697,7 @@ static bool h16_take_spare(struct ane_h16_session *s, size_t size, bool write, for (i = 0; i < s->spare_count; i++) { if (s->spares[i].surface.size != size || s->spares[i].write != write) continue; - *out = s->spares[i].surface; - s->spares[i] = s->spares[--s->spare_count]; + *out = h16_spare_remove(s, i); /* Zeroed like new memory; LOAD writes it back after filling it. */ memset(out->cpu, 0, size); return true; @@ -660,12 +708,20 @@ static bool h16_take_spare(struct ane_h16_session *s, size_t size, bool write, static void h16_give_spare(struct ane_h16_session *s, struct ane_fw_surface *surface, bool write) { - /* A full table keeps the surface mapped and unused until reset. */ - if (surface->cpu && s->spare_count < ANE_H16_SPARES) + if (!surface->cpu) { + memset(surface, 0, sizeof(*surface)); + return; + } + if (s->spare_count < ANE_H16_SPARES) { s->spares[s->spare_count++] = (struct ane_h16_spare) { .surface = *surface, .write = write, }; + s->spare_bytes += surface->size; + } else if (h16_can_unmap(s)) { + s->ops->unmap(s->ctx, surface); + } + /* Otherwise a full table keeps the surface mapped and unused until reset. */ memset(surface, 0, sizeof(*surface)); } @@ -684,6 +740,7 @@ static void h16_export_work(struct work_struct *work) h16_give_spare(s, &e->surface, e->write); kfree(e); } + h16_spares_trim(s, ANE_H16_SPARES_KEEP, ANE_H16_SPARE_BYTES); mutex_unlock(s->lock); } @@ -775,13 +832,22 @@ static void h16_give_back(struct ane_h16_session *s, struct ane_h16_program *p) h16_give_spare(s, &p->aux[i], false); memset(p->surfaces, 0, sizeof(p->surfaces)); p->released = true; + h16_spares_trim(s, ANE_H16_SPARES_KEEP, ANE_H16_SPARE_BYTES); } static int h16_map(struct ane_h16_session *s, size_t size, bool write, struct ane_fw_surface *out) { - int ret = h16_take_spare(s, size, write, out) ? 0 : s->ops->map(s->ctx, size, write, out); + int ret = 0; + if (!h16_take_spare(s, size, write, out)) { + ret = s->ops->map(s->ctx, size, write, out); + /* No DVA range or memory left: free every spare and try once more. */ + if ((ret == -ENOSPC || ret == -ENOMEM) && s->spare_count && h16_can_unmap(s)) { + h16_spares_trim(s, 0, 0); + ret = s->ops->map(s->ctx, size, write, out); + } + } if (ret) return ret; if (!ane_fw_surface_valid(out) || out->size != size || !IS_ALIGNED(out->dva, SZ_16K)) @@ -904,6 +970,19 @@ static int h16_program_map(struct ane_h16_session *s, struct ane_h16_program *p, return ane_h16_custom_bars(&p->snapshot, hwx, s->dsid, p->bars, &p->bar_count); } +/* + * LOAD or CREATE failed with the firmware holding the program, maybe: the + * session is quarantined and the program is kept, memory and all, with + * those closed while quarantined, until a verified power cycle gives them + * back (ane_h16_session_recovered()). + */ +static int h16_failed_load(struct ane_h16_session *s, struct ane_h16_program *p, int error) +{ + p->closed = true; + list_add_tail(&p->node, &s->held); + return h16_fail(s, error); +} + static const size_t h16_load_sequence[] = { ANE_H16_LOAD_SIZE, ANE_H16_CREATE_SIZE, ANE_H16_TERMINATE_SIZE, ANE_H16_UNLOAD_SIZE, }; @@ -915,8 +994,9 @@ static const size_t h16_load_sequence[] = { * queues the first EXECUTE whenever the engine last ran another program, the * first one after boot included (see ane_h16_session_execute()). A LOAD is * serialized with in-flight EXECUTEs. The transport also requeues on every - * program or process change, so a session may hold several programs. Every - * program's firmware-visible memory stays mapped until reset. + * program or process change, so a session may hold several programs. A + * program's firmware-visible memory goes back to the spares once the + * firmware no longer uses it (see h16_give_back()). */ int ane_h16_session_load(struct ane_h16_session *s, struct ane_h16_hwx_snapshot *snapshot, @@ -973,9 +1053,9 @@ int ane_h16_session_load(struct ane_h16_session *s, if (ret) goto free; /* - * From the first mapping on, firmware-visible memory exists that is never - * released before reset, so the program becomes the session's here and - * is never freed. + * From the first mapping on, firmware-visible memory exists, so the + * program becomes the session's here; it is freed only once that memory + * went back (ane_h16_session_put()). */ p->generation = ++s->generation; p->epoch = s->epoch; @@ -988,19 +1068,22 @@ int ane_h16_session_load(struct ane_h16_session *s, if (!ret) ret = ane_h16_load_encode(command, ANE_H16_LOAD_SIZE, p->slots); if (ret) { - /* - * Firmware never saw this program. It keeps its mappings until - * reset, but it must not hold the session against the next LOAD. - */ + /* Firmware never saw this program: its memory goes back now. */ p->closed = true; + h16_give_back(s, p); + ane_h16_session_put(s, p); goto out; } + if (list_empty(&s->programs) && list_empty(&s->held) && !s->running) { + s->pool_base = ane_fw_transport_mark(s->transport); + s->pool_base_set = true; + } ret = h16_send(s, ANE_FW_IO, command, ANE_H16_LOAD_SIZE, &reply); if (!ret) ret = ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE, command, &p->key.program); if (ret) { s->ops->log(s->ctx, "LOAD refused", (u64)-ret, 0); - ret = h16_fail(s, ret); + ret = h16_failed_load(s, p, ret); goto out; } s->ops->log(s->ctx, "LOAD program", p->key.program, p->bar_count); @@ -1012,7 +1095,7 @@ int ane_h16_session_load(struct ane_h16_session *s, &p->key.process); if (ret) { s->ops->log(s->ctx, "CREATE refused", (u64)-ret, 0); - ret = h16_fail(s, ret); + ret = h16_failed_load(s, p, ret); goto out; } /* Each validated program has exactly one procedure. */ @@ -1022,6 +1105,7 @@ int ane_h16_session_load(struct ane_h16_session *s, s->ops->log(s->ctx, "CREATE process", p->key.program, p->key.process); *program = p; out: + h16_spares_trim(s, ANE_H16_SPARES_KEEP, ANE_H16_SPARE_BYTES); kfree(command); return ret; free: @@ -1772,6 +1856,7 @@ int ane_h16_session_close(struct ane_h16_session *s, struct ane_h16_program *p) * reads or writes this program's memory: later programs may reuse it. */ h16_give_back(s, p); + h16_pool_rewind(s); return 0; } EXPORT_SYMBOL_GPL(ane_h16_session_close); @@ -1850,6 +1935,8 @@ void ane_h16_session_restart(struct ane_h16_session *s) { s->epoch++; s->program = NULL; + /* The new firmware's arena starts afresh. */ + s->pool_base_set = false; /* A firmware booted again from the preload starts with gating on. */ s->gate_held = false; s->executed = 0; diff --git a/drivers/accel/ane/ane_h16_program.h b/drivers/accel/ane/ane_h16_program.h index c22919a79349c..d4ca6a37e3023 100644 --- a/drivers/accel/ane/ane_h16_program.h +++ b/drivers/accel/ane/ane_h16_program.h @@ -150,10 +150,22 @@ int ane_h16_custom_bars(const struct ane_h16_hwx_snapshot *snapshot, const struct ane_h16_hwx *hwx, u32 dsid, struct ane_h16_custom_bar *bars, unsigned int *count); -/* Private, pinned mappings in the firmware domain. Never released before reset. */ +/* + * Private, pinned mappings in the firmware domain. Without unmap they are + * never released before reset. + */ struct ane_h16_session_ops { int (*map)(void *ctx, size_t size, bool device_write, struct ane_fw_surface *surface); + /* + * Optional. Give back a surface map() returned: unmap it from the + * engine, free its memory and its DVA range. The session calls it + * only for memory the firmware no longer uses (UNLOAD acknowledged, + * the program lost to a new firmware, or never seen by the firmware), + * only while the engine is up (not asleep, not parked), and with the + * session lock held. + */ + void (*unmap)(void *ctx, const struct ane_fw_surface *surface); void (*log)(void *ctx, const char *item, u64 a, u64 b); /* * Optional. For a surface with a handle: before the engine uses it @@ -286,11 +298,15 @@ void ane_h16_exec_stats_add(struct ane_h16_exec_stats *st, */ /* * Program memory given back by a program that the firmware unloaded. It - * stays mapped until reset like every firmware-visible mapping, and serves - * a later program's mapping of the same size and permission instead of a - * new one, so loading the same model again and again uses no new memory. + * serves a later program's mapping of the same size and permission instead + * of a new one, so loading the same model again and again uses no new + * memory. With ops->unmap the oldest spares beyond ANE_H16_SPARES_KEEP + * entries or ANE_H16_SPARE_BYTES are freed, and all of them when a new + * mapping finds no room; without it, they stay mapped until reset. */ #define ANE_H16_SPARES 64 +#define ANE_H16_SPARES_KEEP 32 +#define ANE_H16_SPARE_BYTES SZ_128M struct ane_h16_spare { struct ane_fw_surface surface; @@ -305,8 +321,10 @@ struct ane_h16_session { struct ane_h16_pending_table pending; /* The last loaded program. */ struct ane_h16_program *program; + /* Oldest first. */ struct ane_h16_spare spares[ANE_H16_SPARES]; unsigned int spare_count; + size_t spare_bytes; /* Quiet time that ends serving the SHAREDMALLOC ring after UNLOAD. */ u64 settle_ns; /* @@ -349,6 +367,12 @@ struct ane_h16_session { bool pipelined; /* Reclaim a batch of command buffers only after its last caller ends. */ size_t execute_mark; + /* + * Command arena position at a LOAD with no program alive in this + * firmware boot: once none is again, every command above it is done. + */ + size_t pool_base; + bool pool_base_set; /* Firmware log lines printed after the last failed EXECUTE. */ unsigned int fwlog_lines; bool quarantined; diff --git a/drivers/accel/ane/ane_h16_program_test.c b/drivers/accel/ane/ane_h16_program_test.c index cf494e1a1c0ea..04be0bc362dff 100644 --- a/drivers/accel/ane/ane_h16_program_test.c +++ b/drivers/accel/ane/ane_h16_program_test.c @@ -80,6 +80,13 @@ struct prog_fw { /* ops->hung calls and the error of the last one. */ unsigned int hung; int hung_error; + /* + * Bytes mapped and not unmapped, ops->unmap calls, the most bytes a + * nonzero window lets be mapped at once (-ENOSPC beyond), and the + * number of the map call that fails with -ENOMEM (0: none). + */ + u64 live, window; + unsigned int unmaps, map_fail_at; }; static u64 fw_load(void *ctx, const void *p) @@ -328,6 +335,12 @@ static int fw_map(void *ctx, size_t size, bool write, struct ane_fw_surface *s) { struct prog_fw *f = ctx; + if (f->window && size > f->window - f->live) + return -ENOSPC; + if (f->map_fail_at && f->maps + 1 == f->map_fail_at) { + f->map_fail_at = 0; + return -ENOMEM; + } *s = (struct ane_fw_surface) { .cpu = fw_zalloc(f->test, size), .dva = f->next_dva, @@ -339,9 +352,21 @@ static int fw_map(void *ctx, size_t size, bool write, struct ane_fw_surface *s) f->next_dva += size; f->maps++; f->writable_maps += write; + f->live += size; return 0; } +static void fw_unmap(void *ctx, const struct ane_fw_surface *s) +{ + struct prog_fw *f = ctx; + + KUNIT_EXPECT_FALSE(f->test, f->s.asleep || f->s.parked); + KUNIT_EXPECT_GE(f->test, f->live, (u64)s->size); + f->live -= s->size; + f->unmaps++; + kunit_release_action(f->test, fw_kvfree, s->cpu); +} + static void fw_log(void *ctx, const char *item, u64 a, u64 b) { struct prog_fw *f = ctx; @@ -399,6 +424,16 @@ static void fw_hung(void *ctx, int error) f->hung_error = error; } +/* As fw_session_ops, with a backend that frees what the session gives back. */ +static const struct ane_h16_session_ops fw_session_ops_unmap = { + .map = fw_map, + .unmap = fw_unmap, + .log = fw_log, + .busy = fw_busy, + .sync = fw_sync, + .hung = fw_hung, +}; + /* As fw_session_ops, with a backend that recovers a quarantined session. */ static const struct ane_h16_session_ops fw_session_ops_hung = { .map = fw_map, @@ -2809,7 +2844,277 @@ static void program_hung_recovery_test(struct kunit *test) ane_h16_session_put(&f->s, p); } +/* Load, run, close and free one program. */ +static int fw_cycle(struct prog_fw *f) +{ + struct ane_h16_program *p = NULL; + int ret = fw_load_program(f, &p); + + if (ret) + return ret; + ret = ane_h16_session_execute(&f->s, p); + if (!ret) + ret = ane_h16_session_close(&f->s, p); + if (!ret) + ane_h16_session_put(&f->s, p); + return ret; +} + +/* + * LOAD, CREATE, TERMINATE and UNLOAD of each program used to stay in the + * 256 KiB command arena until the firmware was booted again: about two + * hundred programs in, every LOAD failed with -ENOSPC. Once the last + * program is torn down the arena goes back to where the first LOAD found + * it, and never below a program that is still loaded. + */ +static void program_pool_rewind_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + size_t base, kept; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + base = ane_fw_transport_mark(&f->t); + for (i = 0; i < 1000; i++) { + KUNIT_ASSERT_EQ(test, fw_cycle(f), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_mark(&f->t), base); + } + + /* A program that stays loaded keeps its commands and all after them. */ + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + kept = ane_fw_transport_mark(&f->t); + KUNIT_EXPECT_GT(test, kept, base); + for (i = 0; i < 20; i++) + KUNIT_ASSERT_EQ(test, fw_cycle(f), 0); + KUNIT_EXPECT_GT(test, ane_fw_transport_mark(&f->t), kept); + KUNIT_ASSERT_EQ(test, ane_h16_session_close(&f->s, p), 0); + ane_h16_session_put(&f->s, p); + KUNIT_EXPECT_EQ(test, ane_fw_transport_mark(&f->t), base); + + /* A quarantined session keeps the arena as it is. */ + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + kept = ane_fw_transport_mark(&f->t); + f->s.quarantined = true; + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, p), -EIO); + KUNIT_EXPECT_EQ(test, ane_fw_transport_mark(&f->t), kept); + KUNIT_EXPECT_FALSE(test, list_empty(&f->s.held)); +} + +/* + * Memory a program of another shape gave back: no later program of these + * tests maps a surface of this size, so only freeing it makes room. + */ +static bool fw_foreign_spare(struct prog_fw *f, size_t size) +{ + struct ane_fw_surface surface; + + if (f->s.spare_count == ANE_H16_SPARES || fw_map(f, size, false, &surface)) + return false; + f->s.spares[f->s.spare_count++] = (struct ane_h16_spare) { .surface = surface }; + f->s.spare_bytes += size; + return true; +} + +/* Larger than any region or section of the fixture programs. */ +#define FW_FOREIGN(i) (SZ_16K * (9 + (i) % 3)) +#define FW_WINDOW SZ_16M + +/* + * Programs of ever new shapes leave spares that no later program can use. + * Without ops->unmap a spare table full of them makes every close leave its + * memory mapped and unused, the DVA window fills, and every LOAD fails with + * -ENOSPC from then on, until reset. With it, the session frees the oldest + * spares beyond its limits: thousands of programs load into a window that + * holds a few dozen, with a new shape left behind before each. + */ +static void program_unmap_window_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test), *g = fw_create(test); + unsigned int i; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, g); + f->window = FW_WINDOW; + for (i = 0; i < ANE_H16_SPARES; i++) + KUNIT_ASSERT_TRUE(test, fw_foreign_spare(f, FW_FOREIGN(i))); + for (i = 0; i < 2000 && !ret; i++) + ret = fw_cycle(f); + KUNIT_EXPECT_EQ(test, ret, -ENOSPC); + KUNIT_EXPECT_LT(test, i, 2000U); + KUNIT_EXPECT_EQ(test, f->unmaps, 0U); + + g->s.ops = &fw_session_ops_unmap; + g->window = FW_WINDOW; + for (i = 0; i < ANE_H16_SPARES; i++) + KUNIT_ASSERT_TRUE(test, fw_foreign_spare(g, FW_FOREIGN(i))); + for (i = 0; i < 2000; i++) { + fw_foreign_spare(g, FW_FOREIGN(i)); + KUNIT_ASSERT_EQ(test, fw_cycle(g), 0); + KUNIT_ASSERT_LE(test, g->s.spare_count, ANE_H16_SPARES_KEEP); + } + KUNIT_EXPECT_GE(test, g->unmaps, 2000U); + KUNIT_EXPECT_FALSE(test, g->s.quarantined); + KUNIT_EXPECT_PTR_EQ(test, g->s.program, NULL); + /* What the spares hold is all that stays mapped. */ + KUNIT_EXPECT_EQ(test, g->live, (u64)g->s.spare_bytes); +} + +/* A mapping that finds no room frees every spare and tries again. */ +static void program_unmap_pressure_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->s.ops = &fw_session_ops_unmap; + f->window = SZ_2M; + for (i = 0; fw_foreign_spare(f, FW_FOREIGN(i)); i++) + ; + /* Not even the smallest mapping fits beside them. */ + f->window = f->live; + KUNIT_ASSERT_GT(test, i, 0U); + KUNIT_ASSERT_LT(test, i, ANE_H16_SPARES_KEEP); + KUNIT_ASSERT_EQ(test, fw_cycle(f), 0); + KUNIT_EXPECT_EQ(test, f->unmaps, i); + KUNIT_EXPECT_EQ(test, f->live, (u64)f->s.spare_bytes); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); +} + +static u8 fw_dummy[SZ_16K]; + +/* The spares stay within their byte limit too, freeing the oldest first. */ +static void program_unmap_bytes_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + const size_t half = ANE_H16_SPARE_BYTES / 2; + u64 newest = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->s.ops = &fw_session_ops_unmap; + /* Large spares with no memory behind them; never taken, only freed. */ + for (i = 0; i < 3; i++) { + newest = f->next_dva; + f->s.spares[f->s.spare_count++] = (struct ane_h16_spare) { + .surface = { .cpu = fw_dummy, .dva = newest, .size = half }, + }; + f->s.spare_bytes += half; + f->next_dva += half; + f->live += half; + } + KUNIT_ASSERT_EQ(test, fw_cycle(f), 0); + KUNIT_EXPECT_LE(test, f->s.spare_bytes, (size_t)ANE_H16_SPARE_BYTES); + KUNIT_EXPECT_EQ(test, f->live, (u64)f->s.spare_bytes); + KUNIT_EXPECT_EQ(test, f->unmaps, 2U); + for (i = 0; i < f->s.spare_count; i++) + if (f->s.spares[i].surface.size == half) + KUNIT_EXPECT_EQ(test, f->s.spares[i].surface.dva, newest); +} + +/* + * Nothing is freed while the engine is parked: the DART invalidation of an + * unmap could not reach it. The spares keep the memory until the next LOAD, + * which runs with the engine up. + */ +static void program_unmap_parked_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->s.ops = &fw_session_ops_unmap; + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + for (i = 0; i < ANE_H16_SPARES_KEEP + 8; i++) + KUNIT_ASSERT_TRUE(test, fw_foreign_spare(f, SZ_16K * 3)); + f->s.parked = true; + KUNIT_ASSERT_EQ(test, ane_h16_session_close(&f->s, p), 0); + ane_h16_session_put(&f->s, p); + KUNIT_EXPECT_EQ(test, f->unmaps, 0U); + KUNIT_EXPECT_GT(test, f->s.spare_count, ANE_H16_SPARES_KEEP); + f->s.parked = false; + KUNIT_ASSERT_EQ(test, fw_cycle(f), 0); + KUNIT_EXPECT_GT(test, f->unmaps, 0U); + KUNIT_EXPECT_LE(test, f->s.spare_count, ANE_H16_SPARES_KEEP); +} + +/* + * A LOAD that fails before the firmware saw the program (here, no memory + * for its third mapping) gives back what it mapped and frees the program. + */ +static void program_unmap_failed_map_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->s.ops = &fw_session_ops_unmap; + f->map_fail_at = 3; + KUNIT_EXPECT_EQ(test, fw_load_program(f, &p), -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, p, NULL); + KUNIT_EXPECT_PTR_EQ(test, f->s.program, NULL); + KUNIT_EXPECT_EQ(test, f->count, 0U); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); + KUNIT_EXPECT_EQ(test, f->s.spare_count, 2U); + KUNIT_EXPECT_EQ(test, f->live, (u64)f->s.spare_bytes); + /* The next LOAD takes them. */ + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + KUNIT_EXPECT_EQ(test, f->s.spare_count, 0U); +} + +/* + * A LOAD the firmware refused quarantines the session; the program is held + * with those closed while quarantined, and the verified power cycle of the + * recovery gives its memory back. + */ +static void program_unmap_refused_load_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + struct mutex *lock = kunit_kzalloc(test, sizeof(*lock), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, lock); + mutex_init(lock); + f->s.ops = &fw_session_ops_unmap; + f->mode = FW_LOAD_NAK; + KUNIT_EXPECT_NE(test, fw_load_program(f, &p), 0); + KUNIT_EXPECT_PTR_EQ(test, p, NULL); + KUNIT_EXPECT_TRUE(test, f->s.quarantined); + KUNIT_EXPECT_EQ(test, f->hung, 1U); + KUNIT_EXPECT_FALSE(test, list_empty(&f->s.held)); + KUNIT_EXPECT_EQ(test, f->s.spare_count, 0U); + + f->s.lock = lock; + ane_h16_session_sleep(&f->s, true); + fw_cold_boot(f); + /* The backend's cold boot starts a new transport. */ + memset(f->head, 0, sizeof(f->head)); + KUNIT_ASSERT_EQ(test, ane_fw_transport_init(&f->t, &f->soc, &fw_ops, f, f->ipc, f->pool, + SZ_16K, f->ipc.dva, ANE_FW_TABLE_SIZE), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + f->mode = FW_OK; + ane_h16_session_restart(&f->s); + ane_h16_session_recovered(&f->s); + ane_h16_session_sleep(&f->s, false); + KUNIT_EXPECT_TRUE(test, list_empty(&f->s.held)); + KUNIT_EXPECT_PTR_EQ(test, f->s.program, NULL); + KUNIT_EXPECT_GT(test, f->s.spare_count, 0U); + KUNIT_EXPECT_EQ(test, f->live, (u64)f->s.spare_bytes); + f->s.lock = NULL; + KUNIT_EXPECT_EQ(test, fw_cycle(f), 0); +} + static struct kunit_case program_cases[] = { + KUNIT_CASE_SLOW(program_pool_rewind_test), + KUNIT_CASE_SLOW(program_unmap_window_test), + KUNIT_CASE(program_unmap_pressure_test), + KUNIT_CASE(program_unmap_bytes_test), + KUNIT_CASE(program_unmap_parked_test), + KUNIT_CASE(program_unmap_failed_map_test), + KUNIT_CASE(program_unmap_refused_load_test), KUNIT_CASE(program_hung_recovery_test), KUNIT_CASE(program_pipeline_out_of_order_test), KUNIT_CASE(program_pipeline_same_program_test),