diff --git a/drivers/media/platform/apple/avd/Kconfig b/drivers/media/platform/apple/avd/Kconfig index ed5ad44276c31..d3bb9763b6230 100644 --- a/drivers/media/platform/apple/avd/Kconfig +++ b/drivers/media/platform/apple/avd/Kconfig @@ -24,8 +24,7 @@ config VIDEO_APPLE_AVD_KUNIT_TEST default KUNIT_ALL_TESTS help Build KUnit tests for the Apple video decoder. They exercise the - checks that the H.264 path makes on the slices that userspace - queues, such as finding the slice data after the slice header, - at the limits and one past them. + H.264 slice bounds, frame-slot ownership, and runtime power management, + including suspend gating and resubmission after firmware resume. If unsure, say N. diff --git a/drivers/media/platform/apple/avd/Makefile b/drivers/media/platform/apple/avd/Makefile index 4d1877c414d41..60560dbab592a 100644 --- a/drivers/media/platform/apple/avd/Makefile +++ b/drivers/media/platform/apple/avd/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only -apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-h264.o avd-vp9.o avd-hevc.o \ - avd-av1.o avd-av1-entropymode.o -apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o +apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-pm.o avd-h264.o avd-vp9.o \ + avd-hevc.o avd-av1.o avd-av1-entropymode.o avd-queue.o avd-backend-open.o \ + avd-backend-apple.o avd-backend-queue.o +apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o avd-queue-test.o \ + avd-pm-test.o obj-$(CONFIG_VIDEO_APPLE_AVD) += apple-avd.o diff --git a/drivers/media/platform/apple/avd/avd-apple-protocol.h b/drivers/media/platform/apple/avd/avd-apple-protocol.h new file mode 100644 index 0000000000000..d0b90c5d48e4c --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-apple-protocol.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_APPLE_PROTOCOL_H_ +#define AVD_APPLE_PROTOCOL_H_ + +#include +#include + +#include "avd-queue.h" + +#define AVD_APPLE_INBOX_BASE 0x01482cf8 +#define AVD_APPLE_COMMAND_SIZE 0x60 +#define AVD_APPLE_RESULT_WORDS 20 +#define AVD_APPLE_SHADOW_SIZE 64 + +/* + * [C] Expected address notification, from the MIT V3 trace_avd.py precedent. + * Keep this rule in one place for correction by the Borage capture. + */ +#define AVD_APPLE_NOTIFY_BASE AVD_APPLE_INBOX_BASE + +/* SPEC 5.1: only these session opcodes have proved builders. */ +static inline int +avd_apple_session_command(u32 cmd[24], u32 client, u32 opcode, + u32 property, u32 value) +{ + unsigned int i; + + if (client >= 1024 || (opcode != 0 && opcode != 2 && opcode != 5 && opcode != 6)) + return -EINVAL; + for (i = 0; i < 24; i++) + cmd[i] = 0; + cmd[0] = opcode | (client << 16); + if (opcode == 6) { + cmd[1] = property; + cmd[2] = value; + } + return 0; +} + +static inline int avd_apple_decode_prefix(struct avd_frame *f, u32 client, + u32 selector, u32 work_sequence) +{ + unsigned int i; + + if (f->idx >= AVD_MAX_FRAMES || client >= 1024 || selector >= 16 || work_sequence >= 4) + return -EINVAL; + for (i = 0; i < 24; i++) + f->cmd[i] = 0; + f->cmd[0] = 1 | (selector << 10) | (client << 22); + f->cmd[1] = f->idx | (work_sequence << 16); + f->cmd[3] = f->frame_num; + return 0; +} + +struct avd_apple_reply_layout { + u8 type_word; + u8 type_shift; + u32 type_mask; + u8 client_word; + u8 client_shift; + u32 client_mask; + u8 frame_word; + u8 status_word; + u32 done_type; + bool verified; +}; + +struct avd_apple_reply { + u32 type; + u32 client; + u32 frame; + int status; +}; + +static inline int +avd_apple_parse_reply(const u32 *record, size_t words, + const struct avd_apple_reply_layout *layout, + struct avd_apple_reply *reply) +{ + if (words != AVD_APPLE_RESULT_WORDS) + return -EMSGSIZE; + if (!layout->verified) + return -EOPNOTSUPP; + if (layout->type_word >= words || layout->client_word >= words || + layout->frame_word >= words || layout->status_word >= words || + layout->type_shift >= 32 || layout->client_shift >= 32 || + !layout->type_mask || !layout->client_mask || layout->done_type == 7) + return -EINVAL; + reply->type = (record[layout->type_word] >> layout->type_shift) & layout->type_mask; + reply->client = (record[layout->client_word] >> layout->client_shift) & layout->client_mask; + reply->frame = record[layout->frame_word]; + /* SPEC 4.3: type 7 is parsing/VP done, never final buffer ownership. */ + if (reply->type == 7) + return -EINPROGRESS; + if (reply->type != layout->done_type) + return -ENOMSG; + reply->status = record[layout->status_word] ? -EIO : 0; + return 0; +} + +static inline u32 avd_apple_notification(unsigned int slot) +{ + return AVD_APPLE_NOTIFY_BASE + AVD_APPLE_COMMAND_SIZE * slot; +} + +struct avd_apple_hw { + const char *fw_name; + /* All offsets below are relative to a DT-owned "apple" resource. */ + u32 aperture_origin; + u32 imem; + u32 imem_size; + u32 version_offset; + u32 itcm_start; + u32 itcm_end; + u32 itcm_clear; + u32 itcm_clear_value; + u32 itcm_enable; + u32 itcm_enable_value; + u32 link_setup; + u32 link_setup_value; + u32 link_state; + u32 link_up; + u32 link_up_value; + u32 ready; + u32 notify; + u32 receive; + u32 receive_status; + u32 receive_pending; + u32 receive_ack; + u32 receive_ack_value; + u32 inbox_consumer; + u32 result_base; + struct avd_apple_reply_layout reply; + /* Match-data fact, never a module parameter or a userspace setting. */ + bool verified; +}; + +extern const struct avd_apple_hw avd_borage_hw; +extern const u8 avd_borage_config[64]; + +#endif /* AVD_APPLE_PROTOCOL_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 64b441be90ffb..ba7beea61c842 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -1525,6 +1525,13 @@ static enum avd_image_fmt avd_av1_get_image_fmt(struct avd_ctx *ctx, #undef BIT_DEPTH } +static unsigned int avd_av1_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_av1_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_av1_submit(struct avd_ctx *ctx) { struct avd_av1_ctx *av1_ctx = ctx->priv; @@ -1580,6 +1587,7 @@ const struct avd_coded_fmt_ops avd_av1_fmt_ops = { .stop = avd_av1_stop, .run = avd_av1_run, .submit = avd_av1_submit, + .pp_kicks = avd_av1_pp_kicks, .try_ctrl = avd_av1_try_ctrl, .get_image_fmt = avd_av1_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd-backend-apple.c b/drivers/media/platform/apple/avd/avd-backend-apple.c new file mode 100644 index 0000000000000..aaa06ba8c9ddd --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-apple.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#include "avd.h" +#include "avd-apple-protocol.h" +#include "avd-queue-internal.h" + +/* SPEC 3.2. Unknown field meanings are intentionally not named. */ +const u8 avd_borage_config[64] = { + [0x0a] = 0x55, [0x0b] = 0x01, /* +0x08: 0x01558000 */ + [0x09] = 0x80, + [0x11] = 0x8a, [0x12] = 0x55, [0x13] = 0x01, + [0x19] = 0x8e, [0x1a] = 0x55, [0x1b] = 0x01, + [0x28] = 0x0a, [0x2a] = 0x04, + [0x2c] = 0x04, + [0x38] = 0x08, [0x3a] = 0x10, [0x3b] = 0x01, +}; + +/* + * Source: clean SPEC 3-5, AppleAVD 993.1 / T6040 Borage. The SPEC does + * not supply the ITCM/version register offsets, link register operations or + * reply field layout. Do not substitute V3/open-firmware offsets for them. + * avd-protocol.rst records every unresolved field and its source/capture. + */ +const struct avd_apple_hw avd_borage_hw = { + .fw_name = "apple/avd-apple-t6040.bin", + .verified = false, +}; + +struct avd_apple { + void __iomem *regs; + const struct avd_apple_hw *hw; + const struct firmware *image; + u32 shadow[AVD_APPLE_SHADOW_SIZE]; + unsigned int shadow_read; + unsigned int shadow_write; + unsigned int producer; + u32 version; +}; + +static void avd_apple_release_image(void *data) +{ + struct avd_apple *apple = data; + + release_firmware(apple->image); +} + +static int avd_apple_image_present(struct avd_apple *apple) +{ + const struct avd_apple_hw *hw = apple->hw; + u32 start = readl(apple->regs + hw->itcm_start); + u32 end = readl(apple->regs + hw->itcm_end); + u32 version; + + if (end <= start || end - start < 4 || end - start > hw->imem_size || + hw->version_offset > end - start - 4) + return -ENOENT; + version = readl(apple->regs + hw->imem + hw->version_offset); + if (!version || version == U32_MAX) + return -ENOENT; + apple->version = version; + return 0; +} + +static int avd_apple_probe(struct avd_dev *avd) +{ + const struct avd_apple_hw *hw = avd->variant->apple; + struct avd_apple *apple; + int ret; + + if (!hw || !hw->verified || !hw->reply.verified) { + dev_info(avd->dev, "Apple transport awaits Borage register/reply facts; using open firmware\n"); + return -EOPNOTSUPP; + } + apple = devm_kzalloc(avd->dev, sizeof(*apple), GFP_KERNEL); + if (!apple) + return -ENOMEM; + apple->hw = hw; + apple->regs = devm_platform_ioremap_resource_byname(avd->pdev, "apple"); + if (IS_ERR(apple->regs)) + return PTR_ERR(apple->regs); + /* Firmware bytes remain in vendorfw; this driver never embeds an image. */ + ret = firmware_request_nowarn(&apple->image, hw->fw_name, avd->dev); + if (ret && ret != -ENOENT) + return ret; + ret = devm_add_action_or_reset(avd->dev, avd_apple_release_image, apple); + if (ret) + return ret; + avd->backend_priv = apple; + return 0; +} + +static int avd_apple_load(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + const struct firmware *fw = apple->image; + size_t i, rounded; + + if (!fw) + return -ENOENT; + if (!fw->size || fw->size > hw->imem_size || + hw->imem_size < 4 || hw->version_offset > hw->imem_size - 4) + return -EINVAL; + rounded = ALIGN(fw->size, 4); + if (rounded > hw->imem_size) + return -EFBIG; + writel(hw->itcm_clear_value, apple->regs + hw->itcm_clear); + writel(0, apple->regs + hw->itcm_start); + writel(rounded, apple->regs + hw->itcm_end); + /* Bounded word copies pad the last word without reading past fw->data. */ + for (i = 0; i < rounded; i += 4) { + u8 bytes[4] = {}; + + memcpy(bytes, fw->data + i, min_t(size_t, 4, fw->size - i)); + writel(get_unaligned_le32(bytes), apple->regs + hw->imem + i); + } + writel(hw->itcm_enable_value, apple->regs + hw->itcm_enable); + return avd_apple_image_present(apple); +} + +static int avd_apple_boot(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + u32 value; + int ret; + + ret = avd_apple_image_present(apple); + if (ret) { + ret = avd_apple_load(avd); + if (ret) + return ret; + } else { + /* Suspend stopped the MCPU without clearing the retained image. */ + writel(hw->itcm_enable_value, apple->regs + hw->itcm_enable); + } + dev_info(avd->dev, "Apple MCPU firmware %u.%u.%u (%08x)\n", + apple->version >> 8, (apple->version >> 4) & 0xf, + apple->version & 0xf, apple->version); + /* SPEC 3.2: decode connection, aperture 1, state 4, then LinkUp. */ + writel(hw->link_setup_value, apple->regs + hw->link_setup); + ret = readl_poll_timeout(apple->regs + hw->link_state, value, + value == 4, 10, 10000); + if (ret) + return ret; + writel(hw->link_up_value, apple->regs + hw->link_up); + ret = readl_poll_timeout(apple->regs + hw->ready, value, + value == 1, 10, 10000); + if (!ret) { + apple->producer = 0; + apple->shadow_read = 0; + apple->shadow_write = 0; + } + return ret; +} + +static void avd_apple_shutdown(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + + /* MCPU disable value must be proved together with enable in the ledger. */ + writel(0, apple->regs + apple->hw->itcm_enable); +} + +static unsigned int avd_apple_max_inflight(struct avd_dev *avd) +{ + /* + * Every frame in VP->PP flight owns one hardware instruction FIFO + * (EXEC bits 7:4), so the FIFO count bounds the frames in flight: + * 15 on v3+ tier 1 parts, 7 on T8103 and T8140. + */ + return min_t(unsigned int, AVD_MAX_FRAMES, avd->variant->fifo_slots ?: 1); +} + +static int avd_apple_send(struct avd_dev *avd, const u32 cmd[24]) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + unsigned int consumer = readl(apple->regs + hw->inbox_consumer) & 7; + u32 address = AVD_APPLE_INBOX_BASE + apple->producer * AVD_APPLE_COMMAND_SIZE; + unsigned int i; + + if (avd_inbox_full(apple->producer, consumer)) + return -EBUSY; + for (i = 0; i < 24; i++) + writel(cmd[i], apple->regs + address - hw->aperture_origin + i * 4); + /* MMIO commands and coherent fp payload precede the four-byte doorbell. */ + wmb(); + writel(avd_apple_notification(apple->producer), apple->regs + hw->notify); + apple->producer = (apple->producer + 1) & 7; + return 0; +} + +static int avd_apple_session_start(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 0, 0, 0)) + return -EINVAL; + return avd_apple_send(ctx->dev, cmd); +} + +static void avd_apple_session_stop(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 2, 0, 0)) + return; + if (avd_apple_send(ctx->dev, cmd)) + dev_warn(ctx->dev->dev, "session stop inbox full\n"); +} + +static void avd_apple_abort(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 5, 0, 0)) + return; + /* Core follows abort with reset before returning any DMA buffer. */ + avd_apple_send(ctx->dev, cmd); +} + +static int avd_apple_set_property(struct avd_ctx *ctx, u32 property, u32 value) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 6, property, value)) + return -EINVAL; + return avd_apple_send(ctx->dev, cmd); +} + +static int avd_apple_submit(struct avd_dev *avd, struct avd_frame *f) +{ + const struct avd_codec_frame_ops *ops = f->ctx->coded_fmt_desc->frame_ops; + u32 selector; + int ret; + + if (!ops || !ops->pack_cmd || !f->fp->cpu || f->ctx->client_id >= 1024) + return -EOPNOTSUPP; + /* SPEC 5.2 prefix precedes the codec-specific builder. */ + selector = get_unaligned_le32(f->fp->cpu + 4) & 0xf; + ret = avd_apple_decode_prefix(f, f->ctx->client_id, selector, f->seq & 3); + if (ret) + return ret; + ret = ops->pack_cmd(f->ctx, f); + if (ret) + return ret; + return avd_apple_send(avd, f->cmd); +} + +static irqreturn_t avd_apple_irq(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + unsigned int budget = AVD_APPLE_SHADOW_SIZE; + + while (budget-- && (readl(apple->regs + hw->receive_status) & hw->receive_pending)) { + unsigned int next = (apple->shadow_write + 1) & 63; + + if (next == apple->shadow_read) + break; + apple->shadow[apple->shadow_write] = readl(apple->regs + hw->receive); + apple->shadow_write = next; + writel(hw->receive_ack_value, apple->regs + hw->receive_ack); + } + while (apple->shadow_read != apple->shadow_write) { + u32 record[AVD_APPLE_RESULT_WORDS]; + struct avd_apple_reply reply; + unsigned int i; + int ret; + + /* Result notification indexing must be verified with the layout. */ + u32 index = apple->shadow[apple->shadow_read]; + + apple->shadow_read = (apple->shadow_read + 1) & 63; + if (index >= AVD_APPLE_SHADOW_SIZE) + continue; + for (i = 0; i < AVD_APPLE_RESULT_WORDS; i++) + record[i] = readl(apple->regs + hw->result_base + + index * sizeof(record) + i * 4); + ret = avd_apple_parse_reply(record, ARRAY_SIZE(record), &hw->reply, &reply); + if (ret) + continue; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + if (f->state == AVD_FRAME_SUBMITTED && f->frame_num == reply.frame && + f->ctx->client_id == reply.client) { + dma_rmb(); + avd_frame_complete(avd, f->tag, reply.status); + break; + } + } + } + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_apple_backend = { + .name = "apple-mcpu", + .probe = avd_apple_probe, + .boot = avd_apple_boot, + .shutdown = avd_apple_shutdown, + .max_inflight = avd_apple_max_inflight, + .session_start = avd_apple_session_start, + .session_stop = avd_apple_session_stop, + .submit = avd_apple_submit, + .abort = avd_apple_abort, + .set_property = avd_apple_set_property, + .irq = avd_apple_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c new file mode 100644 index 0000000000000..d2054d06f948b --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Legacy transport split from avd-drv.c; Copyright The Asahi Linux Contributors. */ +#include +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-inst.h" +#include "avd-regs.h" + +/* + * Compatibility transport for firmware without the queued-program ABI. The + * codec's CPU builder emits a per-slot DMA-visible instruction stream. Only + * this backend touches the VP FIFO. Its untagged PP interrupt limits it to one + * frame; the queued avd-fw transport can replace these ops without changing + * buffer ownership in the core. + */ +int avd_open_build_program(struct avd_ctx *ctx) +{ + struct avd_dev *avd = ctx->dev; + struct avd_frame *f = avd->building; + struct avd_job *job = &ctx->job; + size_t count = 1, pos = 0, i, j; + u32 *words, mask, start, fifo; + int ret; + + if (!f || !job->segments) + return -EINVAL; + for (i = 0; i <= job->num; i++) { + if (job->segments[i].num > AVD_MAX_INST || + check_add_overflow(count, job->segments[i].num + !!i, &count)) + return -EOVERFLOW; + } + if (count > SIZE_MAX / sizeof(u32)) + return -EOVERFLOW; + if (f->prog->size < count * sizeof(u32)) { + ret = avd_buf_alloc(avd, f->prog, count * sizeof(u32)); + if (ret) + return ret; + } + mask = job->codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? + AVD_OP_EXEC_REV3_VP9_MASK : 0; + start = AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); + /* The queued transport gives every frame in flight its own FIFO. */ + ctx->fifo_idx = avd_backend_is_qfw(avd) ? f->idx : 0; + fifo = AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx); + f->fifo_iova = ctx->job_fifo ?: ctx->inst.addr; + ctx->job_fifo = 0; + words = f->prog->cpu; + words[pos++] = AVD_OP_EXEC | mask | start | fifo; + for (i = 0; i <= job->num; i++) { + for (j = 0; j < job->segments[i].num; j++) + words[pos++] = job->segments[i].instructions[j]; + if (i) + words[pos++] = AVD_OP_EXEC | mask | fifo | + AVD_OP_EXEC_FLAG_END(i == job->num); + } + f->prog_words = pos; + f->cmd[0] = job->codec; + /* A mailbox retry may outlive the next preparation on this context. */ + f->cmd[1] = avd_backend_is_qfw(avd) && ctx->coded_fmt_desc->ops->pp_kicks ? + ctx->coded_fmt_desc->ops->pp_kicks(ctx) - 1 : 0; + kvfree(job->segments); + job->segments = NULL; + return 0; +} + +static unsigned int avd_open_max_inflight(struct avd_dev *avd) +{ + return 1; +} + +static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) +{ + struct avd_ctx *ctx = f->ctx; + u32 codec = f->cmd[0], vp = 0; + u32 *words = f->prog->cpu; + unsigned int i; + void __iomem *reg; + u32 max; + + if (avd->compat_active) + return -EBUSY; + /* Native frame_ops require firmware-side PP chaining and tagged replies. */ + if (ctx->coded_fmt_desc->frame_ops) + return -EOPNOTSUPP; + if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || + f->prog_words > f->prog->size / sizeof(u32)) + return -EINVAL; + if (!(avd->capabilities & BIT(codec))) + return -EOPNOTSUPP; + for (i = 0; i < codec; i++) + vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[codec]) + vp = avd->variant->vp_slot_base[codec]; + avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); + reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; + avd->compat_active = f; + avd->compat_ctx = ctx; + avd->compat_pp_kicked = false; + max = readl(avd->ctrl + avd->variant->submit_queue_max_offset + vp * 4); + for (i = 0; i < f->prog_words; i++) { + /* + * Never write the VP FIFO register without room in its queue: a + * write the block cannot accept may stall the bus. The old driver + * only looked between segments, and programs exceed the 128-word + * queue. Wait a bounded time for room, then give the frame to + * the watchdog instead of writing. + */ + if (max) { + u32 cur = readl(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4); + + if (cur >= max && + readl_relaxed_poll_timeout(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4, + cur, cur < max, 5, 2000)) { + dev_err_ratelimited(avd->dev, + "VP%u queue full (%u/%u), not writing\n", + vp, cur, max); + f->deadline = jiffies; + return 0; + } + } + writel(words[i], reg); + } + return 0; +} + +static irqreturn_t avd_open_irq(struct avd_dev *avd) +{ + struct avd_frame *f = avd->compat_active; + u32 status; + + if (!avd_mbox_pop(avd, &status)) + return IRQ_HANDLED; + if (!f) { + /* + * The old driver kicked PP for every VP DONE. A VP DONE with no + * frame in flight (its frame already completed on a surplus PP + * DONE of a multi-kick picture) still needs that kick, or the + * next picture's PP is never started and it times out. + */ + if ((status & 0x11000) == 0 && (status & 0x100) && avd->compat_ctx && + avd->compat_ctx->coded_fmt_desc->ops->submit) + avd->compat_ctx->coded_fmt_desc->ops->submit(avd->compat_ctx); + return IRQ_HANDLED; + } + if (status & 0x1000) { + /* + * Only the PP this driver kicked for the active frame can finish + * it. A surplus PP DONE of a multi-kick picture used to complete + * the next frame early, so userspace reused its buffers while the + * engine was still writing them. + */ + if (!avd->compat_pp_kicked) { + dev_dbg_ratelimited(avd->dev, "PP DONE before the frame's VP, ignored\n"); + return IRQ_HANDLED; + } + avd->compat_active = NULL; + avd->compat_pp_kicked = false; + avd_frame_complete(avd, f->tag, 0); + } else if (status & 0x100) { + /* Old firmware only: no request tag and no firmware-side PP kick. */ + if (f->ctx->coded_fmt_desc->ops->submit) + f->ctx->coded_fmt_desc->ops->submit(f->ctx); + avd->compat_pp_kicked = true; + } else { + dev_err_ratelimited(avd->dev, "open firmware error %#x\n", status); + f->deadline = jiffies; + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); + } + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_open_backend = { + .name = "open-compat", + .boot = avd_boot, + .shutdown = avd_shutdown, + .max_inflight = avd_open_max_inflight, + .submit = avd_open_submit, + .irq = avd_open_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; + +static void avd_open_capabilities(struct avd_dev *avd) +{ + static const struct { + const char *name; + u32 bit; + } codecs[] = { + { "h265", AVD_CAPABILITY_HEVC }, + { "h264", AVD_CAPABILITY_H264 }, + { "vp9", AVD_CAPABILITY_VP9 }, + { "av1", AVD_CAPABILITY_AV1 }, + }; + const char *info; + unsigned int i; + + /* Without a CODECS trailer, keep codecs that need a newer firmware hidden. */ + avd->capabilities = avd->variant->capabilities & + ~avd->variant->fw_caps_required; + avd->fw_capabilities = avd->capabilities; + if (!avd->fw || avd->fw->size < 0x40) + return; + info = (const char *)avd->fw->data + avd->fw->size - 0x40; + if (memcmp(info, "CODECS:", 7)) + return; /* fw_capabilities already set above */ + avd->capabilities = 0; + for (i = 0; i < ARRAY_SIZE(codecs); i++) + if (strnstr(info, codecs[i].name, 0x40)) + avd->capabilities |= avd->variant->capabilities & codecs[i].bit; + avd->fw_capabilities = avd->capabilities; +} + +static int avd_open_request_firmware(struct avd_dev *avd) +{ + int ret; + + if (!avd->variant->fw_name_fallback) + return request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); + ret = firmware_request_nowarn(&avd->fw, avd->variant->fw_name, avd->dev); + if (ret != -ENOENT) + return ret; + dev_info(avd->dev, "%s not installed, using %s\n", + avd->variant->fw_name, avd->variant->fw_name_fallback); + return request_firmware(&avd->fw, avd->variant->fw_name_fallback, avd->dev); +} + +int avd_backend_select(struct avd_dev *avd) +{ + int ret; + + /* Apple probe returns -ENOENT only when no usable image is present. */ + if (avd->variant->apple) { + ret = avd_apple_backend.probe(avd); + if (!ret) { + avd->backend = &avd_apple_backend; + avd->capabilities = avd->variant->capabilities; + return 0; + } + if (ret != -ENOENT && ret != -EOPNOTSUPP) + return ret; + } + avd->backend = &avd_open_backend; + ret = avd_open_request_firmware(avd); + if (!ret) + avd_open_capabilities(avd); + if (!ret) + ret = avd_qfw_probe(avd); + return ret; +} + +int avd_backend_boot(struct avd_dev *avd) +{ + int ret; + + if (!avd_backend_is_apple(avd)) { + avd_pm_prepare_piodma(avd); + avd_qfw_prepare_boot(avd); + } + ret = avd->backend->boot(avd); + /* Never switch command ABIs after frames have used this backend. */ + if (ret == -ENOENT && !avd->fw_generation && avd_backend_is_apple(avd)) { + ret = avd_open_request_firmware(avd); + if (!ret) + ret = avd_qfw_probe(avd); + if (ret) + return ret; + avd->backend = &avd_open_backend; + avd_open_capabilities(avd); + avd_pm_prepare_piodma(avd); + avd_qfw_prepare_boot(avd); + ret = avd->backend->boot(avd); + } + if (!ret) + avd_qfw_select(avd); + if (!ret) + avd->fw_generation++; + return ret; +} diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c new file mode 100644 index 0000000000000..7d21921d17229 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Queued transport for the open AVD firmware (avd-fw queue ABI 2). + * + * The codec's CPU builder still produces the per-frame instruction program. + * Instead of writing it word by word into the VP FIFO register and kicking + * the PP from the VP-done interrupt, the host streams the program into a word + * ring in CM3 SRAM and posts one descriptor per frame. The firmware feeds the + * VP FIFO from SRAM, starts PP itself when the VP is done, and posts a tagged + * completion. No PIODMA (and so no second DART stream) is involved. + */ +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-inst.h" +#include "avd-regs.h" + +#define AVDQ_MAGIC 0x41565131 /* "AVQ1" */ +#define AVDQ_VERSION 2 +#define AVDQ_COUNT 16 +#define AVDQ_DESC_BYTES 64 +#define AVDQ_RING_OFFSET 0x8000 +#define AVDQ_INFO 0x7fc0 +/* Firmware private state starts here; the SRAM ring must end before it. */ +#define AVDQ_RING_LIMIT 0x6000 +#define AVDQ_INFO_RP (AVDQ_INFO + 0x38) +#define AVDQ_INFO_WP (AVDQ_INFO + 0x3c) +#define AVDQ_FEAT_SRAM_RING BIT(0) +#define AVDQ_FEAT_MULTI_KICK BIT(1) +#define AVDQ_FEAT_VP_DONE BIT(2) +#define AVDQ_FLAG_VP_DONE BIT(1) +#define AVDQ_VP_NOTIFY 0x54540000 +#define AVDQ_FLAG_SRAM BIT(0) +#define AVDQ_MAX_PP_EXTRA 255 +#define AVDQ_STATE_SUBMITTED 1 +#define AVDQ_STATE_COMPLETE 2 +#define AVDQ_NOTIFY_MASK 0xffff0000 +#define AVDQ_NOTIFY 0x54530000 +/* SRAM addresses as the CM3 doorbell sees them: 0x01000000 + offset from AVD base. */ +#define AVDQ_CM3_APERTURE(off) (0x01000000 + (off)) +#define AVD_REG_MBOX0_STATUS 0x50 +#define AVD_REG_MBOX0_SEND 0x54 +#define AVDQ_MBOX_STATUS_EMPTY BIT(17) +#define AVDQ_STREAM_TIMEOUT_US (1000 * USEC_PER_MSEC) + +/* Descriptor word offsets (bytes), avd-fw docs/queue.md. */ +enum { + D_MAGIC = 0x00, D_TAG = 0x04, D_CODEC = 0x08, D_VP = 0x0c, + D_PROG_LO = 0x10, D_PROG_HI = 0x14, D_WORDS = 0x18, D_KICK = 0x1c, + D_STATE = 0x20, D_STATUS = 0x24, D_CTAG = 0x28, D_FLAGS = 0x2c, + D_PP_EXTRA = 0x30, D_PP_CONT = 0x34, D_RSVD0 = 0x38, D_RSVD1 = 0x3c, +}; + +struct avd_qfw { + void __iomem *sram; + resource_size_t sram_size; + u32 aperture; /* expected CM3 address of SRAM byte 0 */ + u32 ring_off; + u32 ring_words; + u32 wp; /* host write count, free-running */ + u32 fifo_slots; + u32 codec_mask; /* info+0x24, bit n = codec n */ + bool vp_done; + bool usable; /* last boot found a matching ABI 2 image */ +}; + +static void __iomem *avd_qfw_desc(struct avd_qfw *q, unsigned int idx) +{ + return q->sram + AVDQ_RING_OFFSET + idx * AVDQ_DESC_BYTES; +} + +/* Arguments of dev_dbg() are evaluated only when dynamic debug is enabled. */ +void avd_qfw_debug(struct avd_dev *avd, struct avd_frame *f, const char *where) +{ + struct avd_qfw *q = avd->qfw; + void __iomem *d; + + if (!avd_backend_is_qfw(avd)) + return; + d = avd_qfw_desc(q, f->idx); + dev_dbg(avd->dev, + "queue %s: tag %#x slot %u host %u desc %u status %u ctag %#x rp %#x wp %#x mbox0 %#x mbox1 %#x\n", + where, f->tag, f->idx, f->state, readl(d + D_STATE), + readl(d + D_STATUS), readl(d + D_CTAG), + readl(q->sram + AVDQ_INFO_RP), readl(q->sram + AVDQ_INFO_WP), + readl(avd->mbox + AVD_REG_MBOX0_STATUS), + readl(avd->mbox + AVD_REG_MBOX1_STATUS)); +} + +int avd_qfw_probe(struct avd_dev *avd) +{ + struct resource *code, *sram; + struct avd_qfw *q; + + code = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "code"); + sram = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "sram"); + if (!code || !sram || sram->start < code->start || + resource_size(sram) < AVDQ_RING_OFFSET + AVDQ_COUNT * AVDQ_DESC_BYTES) + return 0; /* no queue mode on this node: legacy transport only */ + q = devm_kzalloc(avd->dev, sizeof(*q), GFP_KERNEL); + if (!q) + return -ENOMEM; + q->sram = devm_ioremap_resource(avd->dev, sram); + if (IS_ERR(q->sram)) + return PTR_ERR(q->sram); + q->sram_size = resource_size(sram); + /* The code window sits at +0x80000 from the decoder base on every SoC. */ + q->aperture = AVDQ_CM3_APERTURE(0x80000 + (u32)(sram->start - code->start)); + avd->qfw = q; + return 0; +} + +/* Read the firmware's info block after boot; true when ABI 2 queue mode fits. */ +static bool avd_qfw_check(struct avd_dev *avd) +{ + struct avd_qfw *q = avd->qfw; + u32 info[16]; + unsigned int i; + + if (!q) + return false; + for (i = 0; i < ARRAY_SIZE(info); i++) + info[i] = readl(q->sram + AVDQ_INFO + 4 * i); + q->usable = false; + q->vp_done = false; + if (info[0] != AVDQ_MAGIC) + return false; + if (info[1] != AVDQ_VERSION || info[2] != AVDQ_COUNT || + info[3] != AVDQ_DESC_BYTES || info[5] != AVDQ_RING_OFFSET || + !(info[11] & AVDQ_FEAT_SRAM_RING) || + !(info[11] & AVDQ_FEAT_MULTI_KICK)) { + dev_info_once(avd->dev, "open firmware queue ABI %u not supported, using legacy transport\n", + info[1]); + return false; + } + if (info[4] != q->aperture) { + dev_warn_once(avd->dev, "firmware SRAM aperture %#x, node needs %#x: wrong image tier, using legacy transport\n", + info[4], q->aperture); + return false; + } + if (info[13] < 1024 || info[12] % 4 || + (u64)info[12] + 4ULL * info[13] > AVDQ_RING_LIMIT || + !info[10] || info[10] > AVDQ_COUNT || info[14] || info[15]) { + dev_warn_once(avd->dev, "firmware SRAM ring geometry invalid, using legacy transport\n"); + return false; + } + q->vp_done = info[11] & AVDQ_FEAT_VP_DONE; + q->ring_off = info[12]; + q->ring_words = info[13]; + q->fifo_slots = min_t(u32, info[10], avd->variant->fifo_slots ?: 1); + q->codec_mask = info[9]; + q->wp = 0; + q->usable = true; + dev_dbg(avd->dev, "queue ABI %u: aperture %#x ring %#x/%u words FIFO slots %u codecs %#x\n", + info[1], info[4], q->ring_off, q->ring_words, q->fifo_slots, q->codec_mask); + return true; +} + +static int avd_qfw_boot(struct avd_dev *avd) +{ + int ret = avd_boot(avd); + + if (ret) + return ret; + /* AP -> CM3 doorbells use mailbox 0; completions stay on mailbox 1. */ + writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); + return 0; +} + +static unsigned int avd_qfw_max_inflight(struct avd_dev *avd) +{ + /* Each frame in flight owns one hardware instruction FIFO (EXEC 7:4). */ + return min_t(unsigned int, AVD_MAX_FRAMES, avd->qfw->fifo_slots); +} + +static void avd_qfw_fail_all(struct avd_dev *avd) +{ + unsigned int i; + + /* ABI: any fatal word or I/O status needs a whole-queue reset. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state == AVD_FRAME_SUBMITTED) + avd->frames[i].deadline = jiffies; + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); +} + +static int avd_qfw_stream(struct avd_dev *avd, const u32 *words, u32 count) +{ + struct avd_qfw *q = avd->qfw; + unsigned long timeout = jiffies + usecs_to_jiffies(AVDQ_STREAM_TIMEOUT_US); + u32 done = 0; + + while (done < count) { + u32 rp = readl(q->sram + AVDQ_INFO_RP); + u32 used = q->wp - rp, space, pos, chunk; + + if (used > q->ring_words) + return -EIO; + space = q->ring_words - used; + if (!space) { + if (time_after(jiffies, timeout)) + return -ETIMEDOUT; + usleep_range(20, 50); + continue; + } + pos = q->wp % q->ring_words; + chunk = min3(space, count - done, q->ring_words - pos); + /* The free-running u32 counter can wrap before the word ring does. */ + chunk = min_t(u64, chunk, (1ULL << 32) - q->wp); + /* + * Word stores only: memcpy_toio() would use 64-bit and byte + * stores, and the CM3 data SRAM is only ever accessed with + * 32-bit ones elsewhere (info block, descriptors). + */ + __iowrite32_copy(q->sram + q->ring_off + 4 * pos, words + done, chunk); + q->wp += chunk; + done += chunk; + /* Words before the count that publishes them. */ + wmb(); + writel(q->wp, q->sram + AVDQ_INFO_WP); + } + return 0; +} + +static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) +{ + struct avd_qfw *q = avd->qfw; + struct avd_ctx *ctx = f->ctx; + u32 codec = f->cmd[0], vp = 0, kick, cont, mask, extra = f->cmd[1]; + u32 *words = f->prog->cpu; + void __iomem *d = avd_qfw_desc(q, f->idx); + unsigned int i; + int ret; + + if (ctx->coded_fmt_desc->frame_ops) + return -EOPNOTSUPP; + if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || + f->prog_words > f->prog->size / sizeof(u32)) + return -EINVAL; + if (!(avd->capabilities & BIT(codec))) + return -EOPNOTSUPP; + /* The program's EXEC words name FIFO f->idx; it must be a real FIFO. */ + if (f->idx >= q->fifo_slots) + return -EINVAL; + /* The prepared picture owns this FIFO, including across inbox retries. */ + if (!f->fifo_iova) + return -EINVAL; + /* A busy inbox owns nothing; retry this same prepared frame later. */ + if (!(readl(avd->mbox + AVD_REG_MBOX0_STATUS) & AVDQ_MBOX_STATUS_EMPTY)) + return -EBUSY; + for (i = 0; i < codec; i++) + vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[codec]) + vp = avd->variant->vp_slot_base[codec]; + if (extra > AVDQ_MAX_PP_EXTRA) + return -E2BIG; + + mask = codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? + AVD_OP_EXEC_REV3_VP9_MASK : 0; + cont = AVD_OP_EXEC | mask | AVD_OP_EXEC_FIFO_IDX(f->idx) | + AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots); + kick = cont | AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3 && + codec != AVD_CODEC_AV1); + + /* FIFO registers of other live frames are never touched. */ + avd->variant->configure_stream(avd, f->fifo_iova, f->idx, vp); + + writel(AVDQ_MAGIC, d + D_MAGIC); + writel(f->tag, d + D_TAG); + writel(codec, d + D_CODEC); + writel(vp, d + D_VP); + writel(q->wp, d + D_PROG_LO); + writel(0, d + D_PROG_HI); + writel(f->prog_words, d + D_WORDS); + writel(kick, d + D_KICK); + writel(0, d + D_STATUS); + writel(0, d + D_CTAG); + writel(AVDQ_FLAG_SRAM | (q->vp_done && ctx->coded_fmt_desc->ops->vp_done ? + AVDQ_FLAG_VP_DONE : 0), d + D_FLAGS); + writel(extra, d + D_PP_EXTRA); + writel(extra ? cont : 0, d + D_PP_CONT); + writel(0, d + D_RSVD0); + writel(0, d + D_RSVD1); + writel(AVDQ_STATE_SUBMITTED, d + D_STATE); + /* Descriptor before the doorbell that hands it to the firmware. */ + wmb(); + (void)readl(d + D_STATE); /* flush posted aperture writes */ + avd_qfw_debug(avd, f, "doorbell"); + dev_dbg(avd->dev, "queue program: tag %#x vp %u fifo %pad words %zu kick %#x start %#x end %#x\n", + f->tag, vp, &f->fifo_iova, f->prog_words, kick, words[0], + words[f->prog_words - 1]); + writel(q->aperture + AVDQ_RING_OFFSET + f->idx * AVDQ_DESC_BYTES, + avd->mbox + AVD_REG_MBOX0_SEND); + + /* The firmware may start feeding while the rest of the program streams in. */ + ret = avd_qfw_stream(avd, words, f->prog_words); + avd_qfw_debug(avd, f, "streamed"); + if (ret) { + dev_err_ratelimited(avd->dev, "queue: program stream failed: %d\n", ret); + /* The descriptor is already posted: only a reset releases it. */ + f->deadline = jiffies; + } + return 0; +} + +static irqreturn_t avd_qfw_irq(struct avd_dev *avd) +{ + struct avd_qfw *q = avd->qfw; + unsigned int n; + u32 word; + + /* At most one VP and PP token per slot, plus one fatal token. */ + for (n = 0; n < 2 * AVDQ_COUNT + 1 && avd_mbox_pop(avd, &word); n++) { + struct avd_frame *f; + void __iomem *d; + u32 state, status, ctag; + unsigned int idx; + + if ((word & AVDQ_NOTIFY_MASK) == AVDQ_VP_NOTIFY && q->vp_done) { + idx = word & 15; + d = avd_qfw_desc(q, idx); + /* Separate record: PP completion may already have been published. */ + state = readl(d + D_RSVD1); + ctag = readl(d + D_RSVD0); + f = &avd->frames[idx]; + if (state == 1 && !((ctag ^ word) & 0xfff0) && + f->state == AVD_FRAME_SUBMITTED && ctag == f->tag) + avd_frame_vp_complete(avd, ctag); + continue; + } + if ((word & AVDQ_NOTIFY_MASK) != AVDQ_NOTIFY) { + dev_err_ratelimited(avd->dev, + "queue: firmware reported %#x, resetting\n", word); + avd_qfw_fail_all(avd); + continue; + } + idx = word & 15; + d = avd_qfw_desc(q, idx); + state = readl(d + D_STATE); + status = readl(d + D_STATUS); + ctag = readl(d + D_CTAG); + f = &avd->frames[idx]; + avd_qfw_debug(avd, f, "completion"); + if (state != AVDQ_STATE_COMPLETE || ((ctag ^ word) & 0xfff0) || + f->state != AVD_FRAME_SUBMITTED || ctag != f->tag) { + dev_warn_ratelimited(avd->dev, "queue: stale completion %#x (state %u tag %#x)\n", + word, state, ctag); + continue; + } + if (status > 1) { + /* An I/O error is not proof of quiescence, even for one frame. */ + avd_qfw_fail_all(avd); + continue; + } + writel(0, d + D_STATE); + avd_frame_complete(avd, ctag, status ? -EINVAL : 0); + } + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_qfw_backend = { + .name = "open-queue", + .boot = avd_qfw_boot, + .shutdown = avd_shutdown, + .max_inflight = avd_qfw_max_inflight, + .submit = avd_qfw_submit, + .irq = avd_qfw_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; + +bool avd_backend_is_qfw(const struct avd_dev *avd) +{ + return avd->backend == &avd_qfw_backend; +} + +bool avd_qfw_has_vp_done(const struct avd_dev *avd) +{ + return avd_backend_is_qfw(avd) && avd->qfw->vp_done; +} + +/* + * SRAM survives a CM3 reload, so a stale info block from an earlier ABI 2 + * image could make a legacy image look queue capable. Clear the magic before + * every open-firmware boot; only an image that publishes it again qualifies. + */ +void avd_qfw_prepare_boot(struct avd_dev *avd) +{ + if (avd->qfw) + writel(0, avd->qfw->sram + AVDQ_INFO); +} + +/* + * Called after every successful open-firmware boot (probe, resume, reset), + * with no frame owned by the firmware. Picks the queued transport when the + * loaded image speaks ABI 2 for this node, else the legacy one. + */ +void avd_qfw_select(struct avd_dev *avd) +{ + const struct avd_backend_ops *old = avd->backend; + + if (old != &avd_open_backend && old != &avd_qfw_backend) + return; + if (avd_qfw_check(avd)) { + /* Legacy boot left mailbox 0 disabled. */ + writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); + avd->backend = &avd_qfw_backend; + /* + * Codecs held back for a newer firmware (T6040 HEVC) are + * enabled by a queue image that lists them in its codec mask. + */ + avd->capabilities = avd->fw_capabilities | + (avd->variant->capabilities & avd->qfw->codec_mask); + } else { + avd->backend = &avd_open_backend; + avd->capabilities = avd->fw_capabilities; + } + if (avd->backend != old) + dev_info(avd->dev, "using %s transport (%u frames in flight)\n", + avd->backend->name, avd->backend->max_inflight(avd)); +} diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 4efd1bd5e2558..d6042c3fb1e92 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include @@ -25,8 +24,7 @@ #include "avd.h" #include "avd-inst.h" #include "avd-regs.h" - -#define AVD_CODEC_INFO_MAX_SIZE 0x40 +#include "avd-apple-protocol.h" static void calc_tile_meta(u32 w, u32 h, u32 bpb, u32 tile_dim, u32 meta_hdr_bytes, u32 *tile, u32 *meta) @@ -76,31 +74,6 @@ void fill_comp(struct avd_comp *comp, enum avd_image_fmt image_fmt, u32 width, comp->size = y_meta + y + uv_meta + uv; } -int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) -{ - /* Keep size equal to the payload length used by the H.264 slice path. */ - avd_buf_free(avd, buf); - - if (size <= 0) - return -ENOMEM; - - buf->size = size; - buf->cpu = - dma_alloc_coherent(avd->dev, buf->size, &buf->addr, GFP_KERNEL); - if (!buf->cpu) { - memset(buf, 0, sizeof(*buf)); - return -ENOMEM; - } - return 0; -} - -void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) -{ - if (buf->cpu) - dma_free_coherent(avd->dev, buf->size, buf->cpu, buf->addr); - memset(buf, 0, sizeof(*buf)); -} - struct avd_decoded_buffer * avd_get_ref_buf(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst, u64 timestamp) { @@ -119,28 +92,6 @@ avd_get_ref_buf(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst, u64 timestamp) return vb2_to_avd_decoded_buf(buf); } -static int avd_wait_submission_queue(struct avd_ctx *ctx, int vp) -{ - struct avd_dev *avd = ctx->dev; - u32 max = readl_relaxed(avd->ctrl + - avd->variant->submit_queue_max_offset + vp * 4); - u32 cur = readl_relaxed( - avd->ctrl + avd->variant->submit_queue_status_offset + vp * 4); - - if (cur == max) { - dev_err(avd->dev, "instruction que full! %d/%d", cur, max); - return 1; - } - - if (cur >= max / 2) { - if (avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) - usleep_range(100, 150); - else - usleep_range(500, 650); - } - return 0; -} - int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) { int ret = 0; @@ -158,467 +109,20 @@ int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) int avd_submit_job(struct avd_ctx *ctx) { - struct avd_dev *avd = ctx->dev; - struct avd_job *sub = &ctx->job; - struct avd_segment *segments, *seg; - unsigned long flags; - size_t num; - int i, idx = 0, vp = 0; - void __iomem *reg; - u32 exec_mask = sub->codec == AVD_CODEC_VP9 && - avd->variant->revision == 3 ? - AVD_OP_EXEC_REV3_VP9_MASK : - 0; - u32 exec_rev_flag = - AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | - AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); - - ctx->fifo_idx = 0; - for (i = 0; i < sub->codec; i++) - vp += avd->variant->vp_slots[i]; - if (avd->variant->vp_slot_base[sub->codec]) - vp = avd->variant->vp_slot_base[sub->codec]; - - /* - * avd_device_run() keeps this job from being finished (and the next one - * from starting) until it returns, and arms the watchdog itself once the - * job belongs to the hardware. Still walk a local copy of the table. - */ - segments = sub->segments; - num = sub->num; - if (WARN_ON_ONCE(!segments)) - return -EINVAL; - - avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); - reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; - - /* the first segment is always the header (needs special handling) */ - - writel(AVD_OP_EXEC | exec_mask | exec_rev_flag | - AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx), - reg); - seg = &segments[idx++]; - for (i = 0; i < seg->num; i++) - writel(seg->instructions[i], reg); - for (; idx <= num; idx++) { - seg = &segments[idx]; - for (i = 0; i < seg->num; i++) - writel(seg->instructions[i], reg); - if ((!(avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) || - sub->codec == AVD_CODEC_HEVC) && - avd_wait_submission_queue(ctx, vp)) - break; - if (idx == num) { - /* A completion from here on belongs to this job. */ - spin_lock_irqsave(&avd->job_lock, flags); - avd->job_end_sent = true; - spin_unlock_irqrestore(&avd->job_lock, flags); - } - writel(AVD_OP_EXEC | exec_mask | - AVD_OP_EXEC_FLAG_END(idx == num), - reg); - } - - if (sub->segments == segments) - sub->segments = NULL; - kvfree(segments); - return 0; -} - -/* Recover at most twice a minute, never twice within this gap. */ -#define AVD_RECOVER_MAX 2 -#define AVD_RECOVER_WINDOW msecs_to_jiffies(60000) -#define AVD_RECOVER_GAP_MS 20 - -/* - * Reset the decoder after a timeout or a VP error. Every reset is a risk - * to the whole machine (the J616s hung hard after timeouts and resets - * three times in one night, and the Neo after a burst of one reset per - * frame), so: stop the interrupt handler and the CM3 first, block the - * decoder's DMA at its DART so a transfer still in flight faults instead - * of reaching memory that may already be reused, give such transfers time - * to drain, and only then assert the pmgr reset. Recover sparingly: a - * third recovery within a minute parks the decoder (DMA blocked) until the - * minute is over, and only a failed reset retires it. - */ -/* - * Parking (resting or retired): stop the CM3 and leave the decoder's DMA - * blocked at its DART, so an engine that is still running (the frame timed - * out, after all) can only fault, never write memory whose buffers are about - * to go back to userspace. - */ -static void avd_park(struct avd_dev *avd) -{ - if (pm_runtime_resume_and_get(avd->dev) < 0) - return; - disable_irq(avd->irq); - avd->irq_parked = true; - avd_shutdown(avd); - if (avd->empty_domain && !avd->dma_parked) { - if (iommu_attach_device(avd->empty_domain, avd->dev)) - dev_err(avd->dev, "retire: could not block DMA\n"); - else - avd->dma_parked = true; - } - pm_runtime_put_autosuspend(avd->dev); -} - -static int avd_reset(struct avd_dev *avd) -{ - int ret = 0; - - if (avd->retired) - return -ENODEV; - if (!avd->recoveries || - time_after(jiffies, avd->recover_window_start + AVD_RECOVER_WINDOW)) { - avd->recover_window_start = jiffies; - avd->recoveries = 0; - } - if (++avd->recoveries > AVD_RECOVER_MAX) { - /* - * Out of budget: stop the decoder with its DMA blocked and let - * it rest until the window ends. Frames fail without touching - * the hardware meanwhile; the next frame after that recovers - * it. Bad streams (or conformance vectors using features the - * hardware lacks) must not cost the user the decoder until a - * reboot. - */ - avd_park(avd); - avd->rest_until = avd->recover_window_start + AVD_RECOVER_WINDOW; - dev_warn(avd->dev, "decoder resting after repeated resets, new frames fail for up to %u s\n", - jiffies_to_msecs(avd->rest_until - jiffies) / 1000 + 1); - return -EBUSY; - } - if (avd->last_recover && - time_before(jiffies, avd->last_recover + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) - msleep(AVD_RECOVER_GAP_MS); - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) - goto retire; - - /* No handler may touch the mailbox while the block is in reset. */ - disable_irq(avd->irq); - dev_warn(avd->dev, "recover: stop firmware\n"); - avd_shutdown(avd); - if (avd->empty_domain) { - dev_warn(avd->dev, "recover: block DMA\n"); - ret = iommu_attach_device(avd->empty_domain, avd->dev); - if (ret) { - dev_err(avd->dev, "recover: DART empty attach failed: %d\n", ret); - goto put; - } - usleep_range(5000, 6000); - } - dev_warn(avd->dev, "recover: reset\n"); - ret = reset_control_reset(avd->rstc); - if (ret) - dev_err(avd->dev, "recover: reset failed: %d\n", ret); - if (avd->empty_domain && ret) { - /* A decoder that failed its reset keeps its DMA blocked. */ - avd->dma_parked = true; - } else if (avd->empty_domain) { - dev_warn(avd->dev, "recover: restore DMA\n"); - iommu_detach_device(avd->empty_domain, avd->dev); - if (avd->domain && iommu_get_domain_for_dev(avd->dev) != avd->domain) { - dev_err(avd->dev, "recover: DART domain not restored\n"); - ret = -EIO; - } - } - if (!ret) { - dev_warn(avd->dev, "recover: boot firmware\n"); - ret = avd_boot(avd); - if (ret) - dev_err(avd->dev, "recover: boot failed: %d\n", ret); - } -put: - avd->last_recover = jiffies ?: 1; - if (!ret) { - enable_irq(avd->irq); - dev_warn(avd->dev, "recover: done\n"); - } - pm_runtime_put_autosuspend(avd->dev); - if (!ret) - return 0; -retire: - avd->retired = true; - dev_err(avd->dev, "decoder retired after a failed reset (%d): reload the module to use it again\n", - ret); - return ret; -} - -/* Bring a rested decoder back: reset with DMA still blocked, restore, boot. */ -static int avd_wake(struct avd_dev *avd) -{ - int ret; - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) - return ret; - dev_warn(avd->dev, "recover: wake after rest\n"); - avd_shutdown(avd); - ret = reset_control_reset(avd->rstc); - if (!ret && avd->dma_parked) { - iommu_detach_device(avd->empty_domain, avd->dev); - if (avd->domain && iommu_get_domain_for_dev(avd->dev) != avd->domain) - ret = -EIO; - else - avd->dma_parked = false; - } - if (!ret) - ret = avd_boot(avd); - if (!ret) { - avd->recoveries = 0; - avd->last_recover = jiffies ?: 1; - if (avd->irq_parked) - enable_irq(avd->irq); - avd->irq_parked = false; - dev_warn(avd->dev, "recover: done\n"); - } else { - avd->retired = true; - dev_err(avd->dev, "decoder retired after a failed wake (%d): reload the module to use it again\n", - ret); - } - pm_runtime_put_autosuspend(avd->dev); - return ret; -} - -static void avd_watchdog_func(struct work_struct *work) -{ - struct avd_dev *avd; - struct avd_ctx *ctx; - unsigned long flags; - bool claimed; - int ret; - ctx = container_of(to_delayed_work(work), struct avd_ctx, - watchdog_work); - if (!ctx) - return; - - avd = ctx->dev; - - /* - * Time out only the job this watchdog was armed for, and only while the - * hardware still owns it. Otherwise that job already finished, and - * resetting the hardware would break whichever job is running now. - */ - spin_lock_irqsave(&avd->job_lock, flags); - claimed = avd->job_ctx == ctx && avd->job_seq == ctx->wd_seq && - avd->job_state == AVD_JOB_SUBMITTED; - if (claimed) - avd->job_state = AVD_JOB_FINISHING; - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: watchdog seq %llu: %s\n", ctx->wd_seq, - claimed ? "timeout" : "stale, ignored"); - if (!claimed) - return; - - dev_err(avd->dev, "Frame processing timed out!"); - - writel(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); - ret = avd_reset(avd); - if (ret) - dev_err(avd->dev, "failed to reset: %d", ret); - /* - * A stream whose frame failed in hardware usually fails the next one - * too (a reset per frame took the Neo down), so stop it here. - */ - ctx->broken = true; - dev_err(avd->dev, "stream stopped after a failed frame; restart streaming to retry\n"); - - avd_job_finish(ctx, VB2_BUF_STATE_ERROR); + return avd_open_build_program(ctx); } static irqreturn_t avd_irq_handler(int irq, void *data) { struct avd_dev *avd = data; - struct avd_ctx *ctx = v4l2_m2m_get_curr_priv(avd->m2m_dev); - - enum vb2_buffer_state state; - unsigned long flags; - bool finish = false, stale = false; - u32 status; - if (!ctx) - return IRQ_HANDLED; - - /* - * Reading RETRIEVE on an empty mailbox returns the previous message - * again. A repeated PP DONE finishes the next frame before its PP has - * run: userspace gets a wrong picture, the frame's own PP DONE never - * comes, and the watchdog resets the decoder. Give a late message - * 10 us, otherwise acknowledge and ignore the interrupt. - */ - if (readl_poll_timeout_atomic(avd->mbox + AVD_REG_MBOX1_STATUS, status, - !(status & AVD_MBOX1_STATUS_EMPTY), 1, 10)) { - writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); - dev_dbg_ratelimited(avd->dev, "spurious interrupt, mailbox empty\n"); - return IRQ_HANDLED; - } - status = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); - - writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); - - if (status & 0x10000) { /* dbg */ - dev_warn(avd->dev, "no handler for IRQ: %3d", status &~0x10000); - writel_relaxed(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); - return IRQ_HANDLED; - } - - if (status & 0x1000) { - /* pp is done ! we are done */ - state = VB2_BUF_STATE_DONE; - } else if (status & 0x100) { - /* a vp is done, kick the pp and hope for the best */ - if(ctx->coded_fmt_desc->ops->submit) - ctx->coded_fmt_desc->ops->submit(ctx); - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx == ctx) - avd->job_vp_done = true; - spin_unlock_irqrestore(&avd->job_lock, flags); - goto done; - } else { - dev_err(avd->dev, "H%d error", status); - /* - * avd-fw err(n): VP n raised its error interrupt and stopped, - * and the PP only starts on a VP done, so the job cannot - * complete any more. Run the watchdog now instead of after its - * timeout: it resets the AVD and fails only this frame, and it - * ignores a job the hardware does not own yet. - */ - mod_delayed_work(system_dfl_wq, &ctx->watchdog_work, 0); - goto done; - } - - /* - * Finish only a job the hardware owns. While avd_device_run() is still - * setting up or submitting, a completion is left over from an older job - * unless this job's END command already went out; then remember it and - * let avd_device_run() finish the job when it returns. - */ - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx != ctx || !avd->job_vp_done) { - /* - * Only the PP kicked for this job's VP can finish it. A surplus - * PP DONE of a multi-kick picture used to finish the next job - * early, so userspace reused its buffers while the decoder was - * still writing them. - */ - stale = true; - } else if (avd->job_state == AVD_JOB_SUBMITTED) { - avd->job_state = AVD_JOB_FINISHING; - finish = true; - } else if (avd->job_state == AVD_JOB_RUNNING && avd->job_end_sent) { - avd->job_pending = true; - avd->job_pending_result = state; - } else { - stale = true; - } - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: irq %#x: %s\n", status, - finish ? "finish" : stale ? "stale, ignored" : "pending"); - if (stale) - dev_warn_ratelimited(avd->dev, "ignoring stale completion"); - if (finish) { - cancel_delayed_work(&ctx->watchdog_work); - avd_job_finish(ctx, state); - } - -done: - return IRQ_HANDLED; -} + /* An interrupt raised before a reset or power-off belongs to that instance. */ + unsigned int gen = READ_ONCE(avd->pm.irq_gen); + irqreturn_t ret; -static void avd_device_run(void *priv) -{ - struct avd_ctx *ctx = priv; - struct avd_dev *avd = ctx->dev; - const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; - enum vb2_buffer_state result = VB2_BUF_STATE_ERROR; - const char *outcome = "already finished"; - unsigned long flags; - bool finish = false, fail; - u64 seq; - int ret; - - if (WARN_ON(!desc)) - return; - - fail = READ_ONCE(avd->retired) || ctx->broken; - /* A rested decoder stays parked until its window ends, then wakes. */ - if (!fail && avd->rest_until) { - if (time_before(jiffies, avd->rest_until) || avd_wake(avd)) - fail = true; - else - avd->rest_until = 0; - } - if (fail) { - struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); - - /* Complete the request's controls too, or its fd never signals. */ - if (src && src->vb2_buf.req_obj.req) - v4l2_ctrl_request_complete(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); - avd_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * IRQ completion cannot wait for an already running watchdog. Drain it - * here, before reusing wd_seq, so an old callback cannot time out this - * context's next job. device_run is sleepable; hold no job_lock here. - */ - cancel_delayed_work_sync(&ctx->watchdog_work); - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) { - avd_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * While the job is RUNNING neither completion IRQs nor the watchdog may - * finish it: v4l2-mem2mem would start the next job on a kworker while - * this one still builds and submits ctx->job. - */ - spin_lock_irqsave(&avd->job_lock, flags); - seq = ++avd->job_seq; - avd->job_ctx = ctx; - avd->job_state = AVD_JOB_RUNNING; - avd->job_end_sent = false; - avd->job_vp_done = false; - avd->job_pending = false; - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: run seq %llu\n", seq); - - ret = desc->ops->run(ctx); - if (ret) { - dev_dbg(avd->dev, "avd-job: run seq %llu failed: %d\n", seq, ret); - avd_job_finish(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * The run either finished the job itself (held capture buffer) or handed - * it to the hardware: arm the watchdog, or finish right away if the - * completion already arrived. - */ - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_seq == seq && avd->job_state == AVD_JOB_RUNNING) { - if (avd->job_pending) { - avd->job_state = AVD_JOB_FINISHING; - result = avd->job_pending_result; - finish = true; - outcome = "completed during submission"; - } else { - avd->job_state = AVD_JOB_SUBMITTED; - ctx->wd_seq = seq; - queue_delayed_work(system_dfl_wq, &ctx->watchdog_work, msecs_to_jiffies(2000)); - outcome = "submitted"; - } - } - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: run seq %llu: %s\n", seq, outcome); - - if (finish) - avd_job_finish(ctx, result); + mutex_lock(&avd->queue_mutex); + ret = avd_pm_irq(avd, gen); + mutex_unlock(&avd->queue_mutex); + return ret; } /* @@ -627,12 +131,7 @@ static void avd_device_run(void *priv) * userspace decode ahead to ride out bitrate spikes the hardware can't * decode in real time (e.g. 150+ Mbit/s 4K60 H.264). */ -static unsigned int max_capture_buffers = 128; -module_param(max_capture_buffers, uint, 0444); -MODULE_PARM_DESC(max_capture_buffers, - "Maximum number of capture buffers per context (default 128)"); - -static int avd_queue_init(void *priv, struct vb2_queue *src_vq, +static int avd_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq) { struct avd_ctx *ctx = priv; @@ -669,7 +168,7 @@ static int avd_queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->lock = &ctx->dev->vdev_lock; dst_vq->dev = ctx->dev->v4l2_dev.dev; dst_vq->allow_cache_hints = true; - dst_vq->max_num_buffers = max(max_capture_buffers, VB2_MAX_FRAME); + dst_vq->max_num_buffers = max(128U, VB2_MAX_FRAME); return vb2_queue_init(dst_vq); } @@ -690,14 +189,15 @@ static int avd_open(struct file *filp) if (ret) goto err_free_ctx; - INIT_DELAYED_WORK(&ctx->watchdog_work, avd_watchdog_func); + INIT_LIST_HEAD(&ctx->queue_node); + ctx->next_dep = AVD_DEP_PREV_DONE; avd_reset_coded_fmt(ctx); avd_reset_decoded_fmt(ctx); v4l2_fh_init(&ctx->fh, video_devdata(filp)); - ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(avd->m2m_dev, ctx, avd_queue_init); + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(avd->m2m_dev, ctx, avd_m2m_queue_init); if (IS_ERR(ctx->fh.m2m_ctx)) { ret = PTR_ERR(ctx->fh.m2m_ctx); goto err_free_ctx; @@ -707,6 +207,7 @@ static int avd_open(struct file *filp) if (ret) goto err_cleanup_m2m_ctx; + avd_queue_add_ctx(ctx); v4l2_fh_add(&ctx->fh, filp); return 0; @@ -725,9 +226,9 @@ static int avd_release(struct file *filp) struct avd_ctx *ctx = file_to_ctx(filp); v4l2_fh_del(&ctx->fh, filp); + avd_queue_del_ctx(ctx); v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); - /* The job has finished; its watchdog must not run on the freed ctx. */ - cancel_delayed_work_sync(&ctx->watchdog_work); + cancel_work_sync(&ctx->submit_work); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); v4l2_fh_exit(&ctx->fh); avd_buf_free(ctx->dev, &ctx->inst); @@ -747,7 +248,9 @@ static const struct v4l2_file_operations avd_fops = { }; static const struct v4l2_m2m_ops avd_m2m_ops = { - .device_run = avd_device_run, + .device_run = avd_queue_run, + .job_ready = avd_queue_ready, + .job_abort = avd_queue_abort, }; static const struct media_device_ops avd_media_ops = { @@ -990,10 +493,12 @@ static const struct avd_variant avd_t8132_variant = { * T6040 (M4 Pro): the T8132 decoder. Its first HEVC job stalled the VP when * it was sent the pre-v5 header (no DMA configuration words, no reserved * scratch slot); T6040 runs the same v5 firmware generation as T8140, so it - * gets the v5 header. HEVC stays hidden from V4L2 clients until that is - * verified on the hardware. + * gets the v5 header. HEVC is advertised only when the loaded firmware + * names h265 in its CODECS trailer. */ static const struct avd_variant avd_t6040_variant = { + .apple = &avd_borage_hw, + .fp_size = 0x1130d0, .vp_slots = { [AVD_CODEC_HEVC] = 4, [AVD_CODEC_H264] = 4, @@ -1002,10 +507,15 @@ static const struct avd_variant avd_t6040_variant = { }, .fifo_slots = 15, .configure_stream = t8122_configure_stream, - .capabilities = AVD_CAPABILITY_H264 | + .capabilities = AVD_CAPABILITY_HEVC | + AVD_CAPABILITY_H264 | AVD_CAPABILITY_VP9 | AVD_CAPABILITY_AV1, - .fw_name = "apple/avd-fw-v5-t1.bin", + /* HEVC only with firmware whose CODECS trailer lists h265. */ + .fw_caps_required = AVD_CAPABILITY_HEVC, + /* v5-t2 matches T6040's SRAM aperture, which the queued transport needs. */ + .fw_name = "apple/avd-fw-v5-t2.bin", + .fw_name_fallback = "apple/avd-fw-v5-t1.bin", .revision = 4, .vp_slot_offset = 0xc, .submit_offset = 0x40, @@ -1053,11 +563,17 @@ static const struct of_device_id avd_of_match[] = { MODULE_DEVICE_TABLE(of, avd_of_match); +static void avd_release_open_firmware(void *data) +{ + struct avd_dev *avd = data; + + release_firmware(avd->fw); +} + static int avd_probe(struct platform_device *pdev) { struct avd_dev *avd; const struct of_device_id *match; - const char *codec_info = NULL; int ret, irq; avd = devm_kzalloc(&pdev->dev, sizeof(*avd), GFP_KERNEL); @@ -1069,12 +585,17 @@ static int avd_probe(struct platform_device *pdev) avd->pdev = pdev; mutex_init(&avd->vdev_lock); - spin_lock_init(&avd->job_lock); + avd_pm_init(avd, NULL, &avd_pm_queue_ops); + ret = avd_queue_init(avd); + if (ret) + return ret; match = of_match_node(avd_of_match, pdev->dev.of_node); avd->variant = match->data; avd->rstc = devm_reset_control_get_exclusive(avd->dev, NULL); + if (IS_ERR(avd->rstc)) + return PTR_ERR(avd->rstc); /* * A node may list a second power domain, "perf": the PMC floor that @@ -1105,6 +626,9 @@ static int avd_probe(struct platform_device *pdev) avd->ctrl = devm_platform_ioremap_resource_byname(pdev, "ctrl"); if (IS_ERR(avd->ctrl)) return PTR_ERR(avd->ctrl); + ret = avd_pm_map_piodma(avd); + if (ret) + return ret; avd->domain = iommu_get_domain_for_dev(avd->dev); if (avd->domain) { @@ -1115,39 +639,16 @@ static int avd_probe(struct platform_device *pdev) } } - ret = request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); - - if (ret) { - dev_err(avd->dev, "failed to load firmware: %d", ret); + ret = avd_backend_select(avd); + if (ret) + return ret; + ret = devm_add_action_or_reset(avd->dev, avd_release_open_firmware, avd); + if (ret) return ret; - } - - /* Determine firmware and HW variant capabilities */ - if (avd->fw->size >= AVD_CODEC_INFO_MAX_SIZE) - codec_info = avd->fw->data + avd->fw->size - AVD_CODEC_INFO_MAX_SIZE; - if (codec_info && strncmp(codec_info, "CODECS:", 7) == 0) { - unsigned int caps = 0; - unsigned int variant_caps = avd->variant->capabilities; - - if (strnstr(codec_info, "av1", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_AV1; - if (strnstr(codec_info, "h264", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_H264; - if (strnstr(codec_info, "h265", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_HEVC; - if (strnstr(codec_info, "vp9", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_VP9; - - dev_info(avd->dev, "Firmware codecs: %.27s\n", codec_info + 7); - - avd->capabilities = caps; - } else { - /* Backward compatibility to old firmware, do not upstream. */ - avd->capabilities = avd->variant->capabilities; - } ret = dma_set_mask_and_coherent(avd->dev, - DMA_BIT_MASK((avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); + DMA_BIT_MASK(avd_backend_is_apple(avd) ? 42 : + (avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); if (ret) { dev_err(avd->dev, "Failed to set DMA mask"); @@ -1157,9 +658,9 @@ static int avd_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 1); if (irq < 0) return irq; - avd->irq = irq; + avd->pm.irq = irq; ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - avd_irq_handler, IRQF_ONESHOT, + avd_irq_handler, IRQF_ONESHOT | IRQF_NO_AUTOEN, dev_name(&pdev->dev), avd); if (ret) { dev_err(avd->dev, "Could not request IRQ 1"); @@ -1176,9 +677,18 @@ static int avd_probe(struct platform_device *pdev) if (ret) return ret; } - pm_runtime_set_autosuspend_delay(avd->dev, 100); - pm_runtime_use_autosuspend(avd->dev); - pm_runtime_enable(avd->dev); + avd_pm_enable(avd); + + /* + * Boot the firmware once now: the transport and the codec list depend + * on what the loaded image reports, and clients enumerate formats and + * size their contexts before the first decode would power it up. + */ + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + dev_warn(avd->dev, "firmware probe boot failed: %d\n", ret); + else + pm_runtime_put_autosuspend(avd->dev); ret = avd_v4l2_init(avd); if (ret) @@ -1195,44 +705,21 @@ static int avd_probe(struct platform_device *pdev) static void avd_remove(struct platform_device *pdev) { struct avd_dev *avd = platform_get_drvdata(pdev); + int ret; - release_firmware(avd->fw); - + v4l2_m2m_suspend(avd->m2m_dev); + ret = avd->pm.queue->drain(avd); + if (ret) + dev_warn(avd->dev, "queue drain failed, resetting before teardown: %d\n", ret); + avd_queue_cleanup(avd); + avd_pm_disable(avd); avd_v4l2_cleanup(avd); - - if (avd->dma_parked) + if (avd->pm.dma_parked) iommu_detach_device(avd->empty_domain, avd->dev); if (avd->empty_domain) iommu_domain_free(avd->empty_domain); - - pm_runtime_disable(avd->dev); - pm_runtime_dont_use_autosuspend(avd->dev); } -static __maybe_unused int avd_runtime_resume(struct device *dev) -{ - int ret; - struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - - ret = avd_boot(avd); - if (ret) - dev_err(dev, "failed to boot"); - return ret; -} - -static __maybe_unused int avd_runtime_suspend(struct device *dev) -{ - struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - - avd_shutdown(avd); - return 0; -} - -static const struct dev_pm_ops avd_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS( - pm_runtime_force_suspend, - pm_runtime_force_resume) SET_RUNTIME_PM_OPS(avd_runtime_suspend, - avd_runtime_resume, NULL) }; - static struct platform_driver avd_driver = { .probe = avd_probe, .remove = avd_remove, diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index e8bc8d0066ead..5ff0026ef3730 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -57,15 +57,18 @@ struct avd_h264_run { u32 slice_data_off; }; -/* state */ -struct avd_h264_ctx { - bool monochrome; - struct avd_h264_reflists { - struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN]; - struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN]; - struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN]; - } reflists; +/* + * One bank per picture in flight. With the queued transport two pictures of + * a stream overlap (VP of one, PP of the other), so everything the hardware + * reads or writes while decoding a picture, and the CPU copies of its slices, + * exists once per bank. The legacy transport uses a single bank. + */ +#define AVD_H264_BANKS 2 +struct avd_bank { + bool monochrome; + bool busy; /* a picture of this bank is not retired yet */ + bool submitted; /* ... and its last slice went to the hardware */ struct avd_buf *slices; struct avd_buf *active_slice; size_t slice_num; @@ -74,6 +77,7 @@ struct avd_h264_ctx { size_t slice_bytes; struct avd_h264_bufs { + /* VP to PP instruction FIFO for pictures of this bank */ struct avd_buf inst; struct avd_buf pipe_state; struct avd_buf above_info; @@ -84,6 +88,20 @@ struct avd_h264_ctx { } bufs; }; +/* state */ +struct avd_h264_ctx { + struct avd_h264_reflists { + struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN]; + struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN]; + struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN]; + } reflists; + + unsigned int nbanks; + struct avd_bank bank[AVD_H264_BANKS]; + /* bank of the picture being built */ + struct avd_bank *b; +}; + /* ffmpeg submits wrong timestamp, use first_mb_in_slice as a workaround */ #define is_new_frame(sl) \ (sl->first_mb_in_slice == 0) /* (ctx->fh.m2m_ctx->new_frame) */ @@ -118,7 +136,7 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_h264_run *run) dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); push(0, ""); - pusha(h264_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); + pusha(h264_ctx->b->bufs.mv_above_info.addr, "mv_above_info", 7); pusha(run->addresses.mv_color, "mv_color", 0); push(0, ""); @@ -272,7 +290,7 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) if (avd->variant->revision == 3) push(0, "zero"); - pusha(h264_ctx->bufs.above_info.addr, "hdr_9c_pps_tile_addr_lsb8", 0); + pusha(h264_ctx->b->bufs.above_info.addr, "hdr_9c_pps_tile_addr_lsb8", 0); push(0, ""); push(0, ""); @@ -280,11 +298,11 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) if (avd->variant->revision == 3) push(0, "zero"); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(h264_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(h264_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(h264_ctx->bufs.ip_above.addr, "ip_above", 1); - pusha(h264_ctx->bufs.lf_above.addr, "lf_above", 2); - pusha(h264_ctx->bufs.lf_above_info.addr, "lf_above_info", 3); + pusha(h264_ctx->b->bufs.ip_above.addr, "ip_above", 1); + pusha(h264_ctx->b->bufs.lf_above.addr, "lf_above", 2); + pusha(h264_ctx->b->bufs.lf_above_info.addr, "lf_above_info", 3); push(0, ""); push_comp(ctx, run->base.comp_out, ctx->comp.offsets); @@ -437,12 +455,12 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) const struct v4l2_ctrl_h264_sps *sps = run->sps; const struct v4l2_ctrl_h264_slice_params *sl = run->slice_params; struct avd_h264_ctx *h264_ctx = ctx->priv; - u32 payload_len = h264_ctx->active_slice->size; + u32 payload_len = h264_ctx->b->active_slice->size; bool en_mode = (pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE) == 0; u32 off = run->slice_data_off; - dma_addr_t slc_a84 = h264_ctx->active_slice->addr + off; + dma_addr_t slc_a84 = h264_ctx->b->active_slice->addr + off; push(AVD_OP_CODED_DATA | AVD_OP_CODED_DATA_BIT_OFF( @@ -538,10 +556,9 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) } } -static int avd_h264_alloc_bufs(struct avd_ctx *ctx) +static int avd_h264_alloc_bufs(struct avd_ctx *ctx, struct avd_bank *bank) { struct avd_dev *dev = ctx->dev; - struct avd_h264_ctx *h264_ctx = ctx->priv; int ret, w, bit_depth, mb; w = fmt_width(ctx); bit_depth = (ctx->image_fmt == AVD_IMG_FMT_420_10BIT || @@ -549,13 +566,13 @@ static int avd_h264_alloc_bufs(struct avd_ctx *ctx) 10 : 8; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(dev, &bank->bufs.inst, fifo_size()); if (ret) { return ret; } if (!(dev->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) { - ret = avd_buf_alloc(dev, &h264_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(dev, &bank->bufs.pipe_state, 0x200); if (ret) { return ret; } @@ -563,24 +580,24 @@ static int avd_h264_alloc_bufs(struct avd_ctx *ctx) mb = DIV_ROUND_UP(w, 16); - ret = avd_buf_alloc(dev, &h264_ctx->bufs.above_info, mb * 20); + ret = avd_buf_alloc(dev, &bank->bufs.above_info, mb * 20); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.ip_above, bit_depth * 4 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.ip_above, bit_depth * 4 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.lf_above, + ret = avd_buf_alloc(dev, &bank->bufs.lf_above, bit_depth * 4 * 4 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.lf_above_info, 32 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.lf_above_info, 32 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.mv_above_info, 32 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.mv_above_info, 32 * mb); if (ret) return ret; @@ -620,35 +637,46 @@ static int avd_h264_validate_sps(struct avd_ctx *ctx, return 0; } -/* Free the slices held for the current picture. */ +/* Free the slices held for one bank's picture. */ +static void avd_h264_drop_bank_slices(struct avd_ctx *ctx, struct avd_bank *bank) +{ + for (size_t i = 0; i < bank->slice_num; i++) + avd_buf_free(ctx->dev, &bank->slices[i]); + bank->slice_num = 0; + bank->slice_bytes = 0; +} + +/* Free the slices held for the picture being built. */ static void avd_h264_drop_slices(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; - for (size_t i = 0; i < h264_ctx->slice_num; i++) - avd_buf_free(ctx->dev, &h264_ctx->slices[i]); - h264_ctx->slice_num = 0; - h264_ctx->slice_bytes = 0; + avd_h264_drop_bank_slices(ctx, h264_ctx->b); } static void avd_h264_stop(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; struct avd_dev *dev = ctx->dev; + unsigned int i; + ctx->max_frames = 1; if (!h264_ctx) return; - avd_buf_free(dev, &h264_ctx->bufs.pipe_state); - avd_buf_free(dev, &h264_ctx->bufs.inst); - avd_buf_free(dev, &h264_ctx->bufs.above_info); - avd_buf_free(dev, &h264_ctx->bufs.lf_above_info); - avd_buf_free(dev, &h264_ctx->bufs.lf_above); - avd_buf_free(dev, &h264_ctx->bufs.ip_above); - avd_buf_free(dev, &h264_ctx->bufs.mv_above_info); - - avd_h264_drop_slices(ctx); - kfree(h264_ctx->slices); + for (i = 0; i < AVD_H264_BANKS; i++) { + struct avd_bank *bank = &h264_ctx->bank[i]; + + avd_buf_free(dev, &bank->bufs.pipe_state); + avd_buf_free(dev, &bank->bufs.inst); + avd_buf_free(dev, &bank->bufs.above_info); + avd_buf_free(dev, &bank->bufs.lf_above_info); + avd_buf_free(dev, &bank->bufs.lf_above); + avd_buf_free(dev, &bank->bufs.ip_above); + avd_buf_free(dev, &bank->bufs.mv_above_info); + avd_h264_drop_bank_slices(ctx, bank); + kfree(bank->slices); + } kfree(h264_ctx); ctx->priv = NULL; @@ -673,20 +701,28 @@ static int avd_h264_start(struct avd_ctx *ctx) return -ENOMEM; ctx->priv = h264_ctx; + /* Overlap pictures only where the firmware chains VP to PP itself. */ + h264_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_H264_BANKS : 1; + h264_ctx->b = &h264_ctx->bank[0]; + + for (unsigned int i = 0; i < h264_ctx->nbanks; i++) { + struct avd_bank *bank = &h264_ctx->bank[i]; + + /* assume one slice per row */ + bank->alloc_slice_num = + ctrl->p_new.p_h264_sps->pic_height_in_map_units_minus1 + 1; + bank->slices = kcalloc(bank->alloc_slice_num, + sizeof(*bank->slices), GFP_KERNEL); + if (!bank->slices) { + ret = -ENOMEM; + goto err_free_ctx; + } - /* assume one slice per row */ - h264_ctx->alloc_slice_num = - ctrl->p_new.p_h264_sps->pic_height_in_map_units_minus1 + 1; - h264_ctx->slices = kcalloc(h264_ctx->alloc_slice_num, - sizeof(*h264_ctx->slices), GFP_KERNEL); - if (!h264_ctx->slices) { - ret = -ENOMEM; - goto err_free_ctx; + ret = avd_h264_alloc_bufs(ctx, bank); + if (ret) + goto err_free_ctx; } - - ret = avd_h264_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; + ctx->max_frames = h264_ctx->nbanks; return 0; @@ -732,22 +768,22 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx, size_t max_slices) void *tmp; size_t alloc_slice_num; - tmp = h264_ctx->slices; + tmp = h264_ctx->b->slices; /* grow by half, and by at least one for a one-row picture */ - alloc_slice_num = h264_ctx->alloc_slice_num + - max_t(size_t, h264_ctx->alloc_slice_num / 2, 1); + alloc_slice_num = h264_ctx->b->alloc_slice_num + + max_t(size_t, h264_ctx->b->alloc_slice_num / 2, 1); alloc_slice_num = min(alloc_slice_num, max_slices); - h264_ctx->slices = kcalloc(alloc_slice_num, sizeof(*h264_ctx->slices), + h264_ctx->b->slices = kcalloc(alloc_slice_num, sizeof(*h264_ctx->b->slices), GFP_KERNEL); - if (!h264_ctx->slices) { + if (!h264_ctx->b->slices) { /* make deallocating a little easier */ - h264_ctx->slices = tmp; + h264_ctx->b->slices = tmp; return -ENOMEM; } - h264_ctx->alloc_slice_num = alloc_slice_num; - memcpy(h264_ctx->slices, tmp, - sizeof(*h264_ctx->slices) * h264_ctx->slice_num); + h264_ctx->b->alloc_slice_num = alloc_slice_num; + memcpy(h264_ctx->b->slices, tmp, + sizeof(*h264_ctx->b->slices) * h264_ctx->b->slice_num); kfree(tmp); /* a slice that arrives before the frame's header has no job table yet */ @@ -853,6 +889,33 @@ VISIBLE_IF_KUNIT bool avd_h264_slice_fits(const struct v4l2_ctrl_h264_sps *sps, avd_h264_slice_budget(decoded_size); } +/* + * Start a new picture in a bank no unretired picture uses, and remember the + * bank in its CAPTURE buffer so retirement frees the right one. + */ +static int avd_h264_pick_bank(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst) +{ + struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_bank *bank = NULL; + unsigned int i; + + /* A picture whose last slice never came: start over in its bank. */ + if (h264_ctx->b->busy && !h264_ctx->b->submitted) + bank = h264_ctx->b; + for (i = 0; !bank && i < h264_ctx->nbanks; i++) + if (!h264_ctx->bank[i].busy) + bank = &h264_ctx->bank[i]; + if (!bank) + return -EBUSY; + avd_h264_drop_bank_slices(ctx, bank); + bank->busy = true; + bank->submitted = false; + h264_ctx->b = bank; + /* 0 means no bank, so a stale or missing value never frees one. */ + vb2_to_avd_decoded_buf(&dst->vb2_buf)->bank = bank - h264_ctx->bank + 1; + return 0; +} + static int avd_h264_run(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; @@ -862,6 +925,14 @@ static int avd_h264_run(struct avd_ctx *ctx) avd_h264_run_preamble(ctx, &run); + if (run.slice_params && is_new_frame(run.slice_params)) { + ret = avd_h264_pick_bank(ctx, run.base.bufs.dst); + if (ret) { + avd_run_postamble(ctx, &run.base); + return ret; + } + } + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); u32 payload_len = vb2_get_plane_payload(&src->vb2_buf, 0); const u8 *data = vb2_plane_vaddr(&src->vb2_buf, 0); @@ -875,8 +946,8 @@ static int avd_h264_run(struct avd_ctx *ctx) return -EINVAL; } - if (!avd_h264_slice_fits(run.sps, h264_ctx->slice_num, - h264_ctx->slice_bytes, payload_len, + if (!avd_h264_slice_fits(run.sps, h264_ctx->b->slice_num, + h264_ctx->b->slice_bytes, payload_len, ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].sizeimage)) { dev_err_ratelimited(ctx->dev->dev, "too many slices for one picture, dropping it\n"); @@ -886,20 +957,20 @@ static int avd_h264_run(struct avd_ctx *ctx) goto err_free_jobs; } - if (h264_ctx->slice_num >= h264_ctx->alloc_slice_num) { + if (h264_ctx->b->slice_num >= h264_ctx->b->alloc_slice_num) { ret = avd_h264_realloc_slices(ctx, avd_h264_max_slices(run.sps)); if (ret) goto err_free_jobs; } - h264_ctx->active_slice = &h264_ctx->slices[h264_ctx->slice_num]; - ret = avd_buf_alloc(ctx->dev, h264_ctx->active_slice, payload_len); + h264_ctx->b->active_slice = &h264_ctx->b->slices[h264_ctx->b->slice_num]; + ret = avd_buf_alloc(ctx->dev, h264_ctx->b->active_slice, payload_len); if (ret) goto err_free_jobs; - memcpy(h264_ctx->active_slice->cpu, data, payload_len); - h264_ctx->slice_num++; - h264_ctx->slice_bytes += PAGE_ALIGN((size_t)payload_len); - h264_ctx->monochrome = run.sps->chroma_format_idc == 0; + memcpy(h264_ctx->b->active_slice->cpu, data, payload_len); + h264_ctx->b->slice_num++; + h264_ctx->b->slice_bytes += PAGE_ALIGN((size_t)payload_len); + h264_ctx->b->monochrome = run.sps->chroma_format_idc == 0; /* Build the P/B{0,1} ref lists. */ v4l2_h264_init_reflist_builder(&reflist_builder, run.decode_params, @@ -923,7 +994,7 @@ static int avd_h264_run(struct avd_ctx *ctx) if (is_new_frame(run.slice_params)) { /* the header segment plus one per slice */ ret = avd_init_job(ctx, AVD_CODEC_H264, - h264_ctx->alloc_slice_num + 1); + h264_ctx->b->alloc_slice_num + 1); if (ret) return ret; stream_hdr(ctx, &run); @@ -940,6 +1011,8 @@ static int avd_h264_run(struct avd_ctx *ctx) return 0; } + ctx->job_fifo = h264_ctx->b->bufs.inst.addr; + h264_ctx->b->submitted = true; return avd_submit_job(ctx); err_free_jobs: @@ -954,22 +1027,32 @@ static void avd_h264_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, enum vb2_buffer_state result) { struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_decoded_buffer *buf; + struct avd_bank *bank; - if (!(src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) { - avd_h264_drop_slices(ctx); + if (src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF || !dst_buf) + return; - /* 4:0:0 has no chroma to decode; the picture is grey */ - if (h264_ctx->monochrome && dst_buf && result == VB2_BUF_STATE_DONE) { - const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; - struct avd_decoded_buffer *dst = - vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); - - /* vb2 must return cache ownership before the CPU writes. */ - dst->grey_chroma_offset = - (size_t)pix->plane_fmt[0].bytesperline * pix->height; - dst->grey_chroma_10bit = - ctx->image_fmt == AVD_IMG_FMT_420_10BIT; - } + buf = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (!buf->bank || buf->bank > h264_ctx->nbanks) + return; + bank = &h264_ctx->bank[buf->bank - 1]; + buf->bank = 0; + avd_h264_drop_bank_slices(ctx, bank); + bank->busy = false; + bank->submitted = false; + + /* 4:0:0 has no chroma to decode; the picture is grey */ + if (bank->monochrome && result == VB2_BUF_STATE_DONE) { + const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; + struct avd_decoded_buffer *dst = + vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + + /* vb2 must return cache ownership before the CPU writes. */ + dst->grey_chroma_offset = + (size_t)pix->plane_fmt[0].bytesperline * pix->height; + dst->grey_chroma_10bit = + ctx->image_fmt == AVD_IMG_FMT_420_10BIT; } } diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 49d8f046ac5c9..b2ce0f3711390 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -79,10 +79,14 @@ struct avd_hevc_run { struct avd_hevc_tile_info tile_info; }; -struct avd_hevc_ctx { - struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix_cache; +/* See avd-h264.c: one bank per picture in flight on the queued transport. */ +#define AVD_HEVC_BANKS 2 +struct avd_hevc_bank { + bool busy; struct avd_h264_bufs { + /* VP to PP instruction FIFO for pictures of this bank */ + struct avd_buf inst; struct avd_buf mv_above_info; struct avd_buf az_above; struct avd_buf ip_above; @@ -91,9 +95,17 @@ struct avd_hevc_ctx { struct avd_buf lf_left; struct avd_buf lf_left_info; struct avd_buf sw_left; - struct avd_buf inst; struct avd_buf pipe_state; } bufs; +}; + +struct avd_hevc_ctx { + struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix_cache; + + unsigned int nbanks; + struct avd_hevc_bank bank[AVD_HEVC_BANKS]; + /* bank of the picture being built */ + struct avd_hevc_bank *b; int submit_num; }; @@ -110,7 +122,7 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_hevc_run *run) push(ctx->dev->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER ? HEVC_DMA_CONFIG : 0, "reference_dma_config"); - pusha(hevc_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); + pusha(hevc_ctx->b->bufs.mv_above_info.addr, "mv_above_info", 7); pusha(run->addresses.mv_color, "mv_color", 0); push(0, ""); @@ -357,30 +369,30 @@ static void set_header(struct avd_ctx *ctx, struct avd_hevc_run *run) if (avd->variant->revision == 3) push(0, "zero"); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(hevc_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(hevc_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(hevc_ctx->bufs.ip_above.addr, "scratch_addr", 0); - pusha(hevc_ctx->bufs.lf_above.addr, "scratch_addr", 1); - pusha(hevc_ctx->bufs.lf_above_info.addr, "scratch_addr", 2); + pusha(hevc_ctx->b->bufs.ip_above.addr, "scratch_addr", 0); + pusha(hevc_ctx->b->bufs.lf_above.addr, "scratch_addr", 1); + pusha(hevc_ctx->b->bufs.lf_above_info.addr, "scratch_addr", 2); if ((pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) || !(avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER)) { - pusha(hevc_ctx->bufs.lf_left.addr, + pusha(hevc_ctx->b->bufs.lf_left.addr, "scratch_addr", 3); - pusha(hevc_ctx->bufs.lf_left_info.addr, + pusha(hevc_ctx->b->bufs.lf_left_info.addr, "scratch_addr", 4); if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) pusha(0, "reserved_scratch", 0); - pusha(hevc_ctx->bufs.az_above.addr, + pusha(hevc_ctx->b->bufs.az_above.addr, "scratch_addr", 8); - pusha(hevc_ctx->bufs.sw_left.addr, + pusha(hevc_ctx->b->bufs.sw_left.addr, "scratch_addr", 9); } else { pusha(0, "", 3); pusha(0, "", 4); if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) pusha(0, "reserved_scratch", 0); - pusha(hevc_ctx->bufs.az_above.addr, + pusha(hevc_ctx->b->bufs.az_above.addr, "scratch_addr", 8); pusha(0, "", 9); } @@ -1056,17 +1068,16 @@ static int avd_hevc_validate_sps(struct avd_ctx *ctx, return 0; } -static int avd_hevc_alloc_bufs(struct avd_ctx *ctx) +static int avd_hevc_alloc_bufs(struct avd_ctx *ctx, struct avd_hevc_bank *bank) { struct avd_dev *avd = ctx->dev; - struct avd_hevc_ctx *hevc_ctx = ctx->priv; int ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(avd, &bank->bufs.inst, fifo_size()); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(avd, &bank->bufs.pipe_state, 0x200); if (ret) return ret; @@ -1077,20 +1088,26 @@ static void avd_hevc_stop(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; + unsigned int i; + ctx->max_frames = 1; if (!hevc_ctx) return; - avd_buf_free(avd, &hevc_ctx->bufs.pipe_state); - avd_buf_free(avd, &hevc_ctx->bufs.inst); - avd_buf_free(avd, &hevc_ctx->bufs.mv_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.az_above); - avd_buf_free(avd, &hevc_ctx->bufs.ip_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left_info); - avd_buf_free(avd, &hevc_ctx->bufs.sw_left); + for (i = 0; i < AVD_HEVC_BANKS; i++) { + struct avd_h264_bufs *bufs = &hevc_ctx->bank[i].bufs; + + avd_buf_free(avd, &bufs->pipe_state); + avd_buf_free(avd, &bufs->inst); + avd_buf_free(avd, &bufs->mv_above_info); + avd_buf_free(avd, &bufs->az_above); + avd_buf_free(avd, &bufs->ip_above); + avd_buf_free(avd, &bufs->lf_above); + avd_buf_free(avd, &bufs->lf_above_info); + avd_buf_free(avd, &bufs->lf_left); + avd_buf_free(avd, &bufs->lf_left_info); + avd_buf_free(avd, &bufs->sw_left); + } kfree(hevc_ctx); ctx->priv = NULL; @@ -1099,6 +1116,7 @@ static void avd_hevc_stop(struct avd_ctx *ctx) static int avd_hevc_start(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx; + unsigned int i; int ret; hevc_ctx = kzalloc(sizeof(*hevc_ctx), GFP_KERNEL); @@ -1106,9 +1124,15 @@ static int avd_hevc_start(struct avd_ctx *ctx) return -ENOMEM; ctx->priv = hevc_ctx; - ret = avd_hevc_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; + /* Overlap pictures only where the firmware chains VP to PP itself. */ + hevc_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_HEVC_BANKS : 1; + hevc_ctx->b = &hevc_ctx->bank[0]; + for (i = 0; i < hevc_ctx->nbanks; i++) { + ret = avd_hevc_alloc_bufs(ctx, &hevc_ctx->bank[i]); + if (ret) + goto err_free_ctx; + } + ctx->max_frames = hevc_ctx->nbanks; return 0; @@ -1117,6 +1141,38 @@ static int avd_hevc_start(struct avd_ctx *ctx) return ret; } +/* Take a bank no unretired picture uses; record it in the CAPTURE buffer. */ +static int avd_hevc_pick_bank(struct avd_ctx *ctx, struct avd_decoded_buffer *dst) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + unsigned int i; + + for (i = 0; i < hevc_ctx->nbanks; i++) { + if (hevc_ctx->bank[i].busy) + continue; + hevc_ctx->bank[i].busy = true; + hevc_ctx->b = &hevc_ctx->bank[i]; + dst->bank = i + 1; + return 0; + } + return -EBUSY; +} + +static void avd_hevc_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, + struct vb2_v4l2_buffer *dst_buf, + enum vb2_buffer_state result) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + struct avd_decoded_buffer *buf; + + if (!dst_buf || !hevc_ctx) + return; + buf = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (buf->bank && buf->bank <= hevc_ctx->nbanks) + hevc_ctx->bank[buf->bank - 1].busy = false; + buf->bank = 0; +} + static int avd_hevc_alloc_scratch(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; @@ -1152,51 +1208,51 @@ static int avd_hevc_alloc_scratch(struct avd_ctx *ctx, struct avd_hevc_run *run) tile_info->row_height[i] : max_row; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.ip_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.ip_above, ((max_col * max_cu_width) / 4) * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.mv_above_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.mv_above_info, ((max_col * max_cu_width) / 16) * 20); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_above, DIV_ROUND_UP(w, 16) * bit_depth * (10 + 6) + (cols - 1) * 256); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_above_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_above_info, DIV_ROUND_UP(w + 7, 16) * 36 + cols * 128); if (ret) return ret; if (pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) { - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_left, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_left, ((max_row * max_cu_width) / 4) * 36 + 144 /* why? */); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_left_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_left_info, ((max_row * max_cu_width) / 4) * 9); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.az_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.az_above, /* not sure if its cols or rows */ DIV_ROUND_UP(w, 64) * 144 + (cols - 1) * 128); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.sw_left, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.sw_left, ((max_row * max_cu_width) / 4) * 216); if (ret) return ret; } else { - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.az_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.az_above, DIV_ROUND_UP(w + 7, 16) * 4 * bit_depth); if (ret) return ret; @@ -1390,6 +1446,7 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) static int avd_hevc_run(struct avd_ctx *ctx) { + struct avd_hevc_ctx *hevc_ctx = ctx->priv; struct avd_hevc_run run = {}; struct avd_decoded_buffer *dst; int ret; @@ -1403,6 +1460,10 @@ static int avd_hevc_run(struct avd_ctx *ctx) dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); update_dec_buf_info(dst, &run.sl[0]); + ret = avd_hevc_pick_bank(ctx, dst); + if (ret) + goto complete; + ret = avd_hevc_compute_tiles(ctx, &run); if (ret) goto complete; @@ -1419,8 +1480,10 @@ static int avd_hevc_run(struct avd_ctx *ctx) set_header(ctx, &run); ret = stream_slices(ctx, &run); avd_run_postamble(ctx, &run.base); - if (!ret) + if (!ret) { + ctx->job_fifo = hevc_ctx->b->bufs.inst.addr; ret = avd_submit_job(ctx); + } goto done; complete: @@ -1442,6 +1505,13 @@ static int avd_hevc_try_ctrl(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl) return 0; } +static unsigned int avd_hevc_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_hevc_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_hevc_submit(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; @@ -1467,7 +1537,9 @@ const struct avd_coded_fmt_ops avd_hevc_fmt_ops = { .start = avd_hevc_start, .stop = avd_hevc_stop, .run = avd_hevc_run, + .done = avd_hevc_done, .submit = avd_hevc_submit, + .pp_kicks = avd_hevc_pp_kicks, .try_ctrl = avd_hevc_try_ctrl, .get_image_fmt = avd_hevc_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd-hw.c b/drivers/media/platform/apple/avd/avd-hw.c index a3987ac4510de..b64c0bf4d78ea 100644 --- a/drivers/media/platform/apple/avd/avd-hw.c +++ b/drivers/media/platform/apple/avd/avd-hw.c @@ -42,9 +42,12 @@ int avd_boot(struct avd_dev *avd) { + struct resource *code = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "code"); u32 val; int ret; + if (!avd->fw || !avd->fw->size || !code || avd->fw->size > resource_size(code)) + return -EINVAL; if (avd->variant->revision != 3) dev_info_once(avd->dev, "booting hw version: %04x", readl_relaxed(avd->ctrl)); @@ -64,6 +67,28 @@ int avd_boot(struct avd_dev *avd) return 0; } +/* + * Pop one firmware message. Reading RETRIEVE on an empty mailbox returns the + * previous message again, and a repeated PP DONE completes the next frame + * before its PP has run. An interrupt can also come just before its message + * is readable, so allow 10 us; after that treat the interrupt as spurious. + * Always acknowledges the interrupt. Returns false when there is no message. + */ +bool avd_mbox_pop(struct avd_dev *avd, u32 *msg) +{ + u32 st; + int ret; + + ret = readl_poll_timeout_atomic(avd->mbox + AVD_REG_MBOX1_STATUS, st, + !(st & AVD_MBOX1_STATUS_EMPTY), 1, 10); + if (!ret) + *msg = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); + writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + if (ret) + dev_dbg_ratelimited(avd->dev, "spurious interrupt, mailbox empty\n"); + return !ret; +} + void avd_shutdown(struct avd_dev *avd) { writel_relaxed(AVD_RUN_CTRL_UNK_STOP, avd->mbox + AVD_REG_RUN_CTRL); diff --git a/drivers/media/platform/apple/avd/avd-inst.h b/drivers/media/platform/apple/avd/avd-inst.h index e9b9ef908b8d5..dcdb205d79315 100644 --- a/drivers/media/platform/apple/avd/avd-inst.h +++ b/drivers/media/platform/apple/avd/avd-inst.h @@ -3,6 +3,7 @@ #ifndef AVD_INST_H_ #define AVD_INST_H_ /* instruction stream things */ +#include #include #include "avd.h" diff --git a/drivers/media/platform/apple/avd/avd-pm-test.c b/drivers/media/platform/apple/avd/avd-pm-test.c new file mode 100644 index 0000000000000..61b911e51c119 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm-test.c @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "avd.h" +#include "avd-regs.h" + +struct avd_pm_test { + struct avd_dev avd; + bool queue_initialized; + struct avd_deferred_unmap unmap; + unsigned int draining; + unsigned int step; + unsigned int unmap_step; + unsigned int boot_step; + unsigned int resubmit_step; + unsigned int shutdowns; + unsigned int submissions; + int boot_error; + int suspend_error; + bool unmapped_active; +}; + +static struct avd_pm_test *avd_pm_fixture(struct avd_dev *avd) +{ + return container_of(avd, struct avd_pm_test, avd); +} + +static bool avd_pm_test_idle(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + return avd_pm_can_suspend(fixture->avd.frames, fixture->draining); +} + +static int avd_pm_test_drain(struct avd_dev *avd) +{ + return avd_pm_test_idle(avd) ? 0 : -EBUSY; +} + +static int avd_pm_test_boot(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->boot_step = ++fixture->step; + return fixture->boot_error; +} + +static void avd_pm_test_shutdown(struct avd_dev *avd) +{ + avd_pm_fixture(avd)->shutdowns++; +} + +static int avd_pm_test_suspend(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + if (fixture->suspend_error) + return fixture->suspend_error; + return avd_pm_backend_suspend(avd); +} + +static int avd_pm_test_submit(struct avd_dev *avd, struct avd_frame *frame) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->resubmit_step = ++fixture->step; + fixture->submissions++; + frame->state = AVD_FRAME_SUBMITTED; + return 0; +} + +static void avd_pm_test_resubmit(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + unsigned int i; + + fixture->resubmit_step = ++fixture->step; + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (fixture->avd.frames[i].state == AVD_FRAME_RESERVED) + avd->backend->submit(avd, &fixture->avd.frames[i]); +} + +static void avd_pm_test_unmap(struct avd_dev *avd, + struct avd_deferred_unmap *unmap) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->unmap_step = ++fixture->step; + fixture->unmapped_active = avd->pm.dart_active; +} + +static const struct avd_queue_pm_ops avd_pm_test_queue = { + .idle = avd_pm_test_idle, + .drain = avd_pm_test_drain, + .resubmit = avd_pm_test_resubmit, +}; + +static const struct avd_backend_ops avd_pm_test_backend = { + .name = "test", + .boot = avd_pm_test_boot, + .shutdown = avd_pm_test_shutdown, + .submit = avd_pm_test_submit, + .suspend = avd_pm_test_suspend, + .resume = avd_pm_backend_resume, +}; + +static int avd_pm_test_init(struct kunit *test) +{ + struct avd_pm_test *fixture; + + fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + if (!fixture) + return -ENOMEM; + test->priv = fixture; + avd_pm_init(&fixture->avd, &avd_pm_test_backend, &avd_pm_test_queue); + INIT_LIST_HEAD(&fixture->unmap.node); + fixture->unmap.unmap = avd_pm_test_unmap; + return 0; +} + +static void avd_pm_suspend_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + unsigned int i, state; + + KUNIT_EXPECT_TRUE(test, avd_pm_test_idle(&fixture->avd)); + fixture->draining = 1; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), -EBUSY); + fixture->draining = 0; + + /* An occupied slot anywhere vetoes idle, even after completion IRQ. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) { + for (state = AVD_FRAME_RESERVED; state <= AVD_FRAME_DONE; state++) { + fixture->avd.frames[i].state = state; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), -EBUSY); + } + fixture->avd.frames[i].state = AVD_FRAME_FREE; + } + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), 0); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); +} + +static void avd_pm_resume_order_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_EQ(test, fixture->unmap_step, 0U); + fixture->avd.frames[0].state = AVD_FRAME_RESERVED; + fixture->avd.frames[1].state = AVD_FRAME_SUBMITTED; + fixture->avd.frames[2].state = AVD_FRAME_DONE; + fixture->avd.frames[15].state = AVD_FRAME_RESERVED; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_TRUE(test, fixture->unmapped_active); + KUNIT_EXPECT_LT(test, fixture->unmap_step, fixture->boot_step); + KUNIT_EXPECT_LT(test, fixture->boot_step, fixture->resubmit_step); + KUNIT_EXPECT_EQ(test, fixture->submissions, 2U); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[0].state, AVD_FRAME_SUBMITTED); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[2].state, AVD_FRAME_DONE); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); +} + +static void avd_pm_resume_failure_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + fixture->avd.frames[0].state = AVD_FRAME_RESERVED; + fixture->boot_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_power_on(avd), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, avd->pm.dart_active); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[0].state, AVD_FRAME_RESERVED); + + /* A later successful resume retries the still-unposted frame once. */ + fixture->boot_error = 0; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_EQ(test, fixture->submissions, 1U); +} + +static void avd_pm_suspend_failure_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + fixture->suspend_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, avd->pm.dart_active); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); +} + +static void avd_pm_deferred_unmap_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + unsigned int step; + + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_TRUE(test, fixture->unmapped_active); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); + KUNIT_ASSERT_EQ(test, avd_pm_power_off(avd), 0); + step = fixture->unmap_step; + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_EQ(test, fixture->unmap_step, step); + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_GT(test, fixture->unmap_step, step); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); +} + +static void avd_pm_real_queue_resume_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct avd_ctx *ctx; + unsigned int i; + + avd->dev = kunit_device_register(test, "avd-pm-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, avd->dev); + KUNIT_ASSERT_EQ(test, avd_queue_init(avd), 0); + fixture->queue_initialized = true; + avd->backend = &avd_pm_test_backend; + avd->pm.queue = &avd_pm_queue_ops; + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->dev = avd; + for (i = 0; i < 3; i++) { + avd->frames[i].ctx = ctx; + avd->frames[i].seq = i + 1; + avd->frames[i].deadline = jiffies + msecs_to_jiffies(60000); + avd->frames[i].state = AVD_FRAME_RESERVED; + } + /* This one was posted before the power transition; never replay it. */ + avd->frames[1].state = AVD_FRAME_SUBMITTED; + avd->inflight = 3; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + flush_delayed_work(&avd->queue_work); + cancel_delayed_work_sync(&avd->queue_work); + KUNIT_EXPECT_EQ(test, fixture->submissions, 2U); + KUNIT_EXPECT_LT(test, fixture->boot_step, fixture->resubmit_step); + KUNIT_EXPECT_EQ(test, avd->frames[0].state, AVD_FRAME_SUBMITTED); + KUNIT_EXPECT_EQ(test, avd->frames[2].state, AVD_FRAME_SUBMITTED); +} + +static void avd_pm_real_queue_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + spin_lock_init(&avd->queue_lock); + avd->pm.queue = &avd_pm_queue_ops; + avd_pm_drain_begin(avd); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -EBUSY); + avd_pm_drain_end(avd); + avd->frames[15].state = AVD_FRAME_DONE; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -EBUSY); + avd->frames[15].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), 0); +} + +static void avd_pm_open_boot_replay_test(struct kunit *test) +{ + static const struct avd_variant variant = { .revision = 3 }; + static const u8 image[] = { 0x12, 0x34, 0x56, 0x78 }; + static const struct firmware fw = { .data = image, .size = sizeof(image) }; + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct resource code = { + .name = "code", .flags = IORESOURCE_MEM, + .start = 0x1000, .end = 0x10ff, + }; + struct platform_device *pdev; + u8 observed[sizeof(image)]; + unsigned int i; + + pdev = kunit_kzalloc(test, sizeof(*pdev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pdev); + pdev->resource = &code; + pdev->num_resources = 1; + avd->pdev = pdev; + avd->variant = &variant; + avd->fw = &fw; + avd->backend = &avd_open_backend; + /* Ordinary test RAM, never a mapped target aperture. */ + avd->code = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); + avd->mbox = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); + avd->pm.piodma = (__force void __iomem *)kunit_kzalloc(test, 0x28, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, avd->code); + KUNIT_ASSERT_NOT_NULL(test, avd->mbox); + KUNIT_ASSERT_NOT_NULL(test, avd->pm.piodma); + avd->pm.sram_phys = 0x123456780ULL; + for (i = 0; i < 2; i++) { + /* Model loss of both ITCM and PIODMA configuration between boots. */ + memset_io(avd->code, 0, sizeof(image)); + writel(0, avd->pm.piodma + 0x24); + /* The fixture supplies firmware readiness; no firmware is executed. */ + writel(1, avd->mbox + AVD_REG_FLAG0_SET); + KUNIT_ASSERT_EQ(test, avd_backend_boot(avd), 0); + KUNIT_EXPECT_EQ(test, readl(avd->pm.piodma + 0x24), 0x12345678U); + memcpy_fromio(observed, avd->code, sizeof(observed)); + KUNIT_EXPECT_MEMEQ(test, observed, image, sizeof(image)); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_RUN_CTRL), + (u32)AVD_RUN_CTRL_UNK_RUN); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX1_STATUS), + (u32)AVD_MBOX_ENABLE); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX_IRQ_ENABLE), + (u32)AVD_MBOX1_NOT_EMPTY); + avd_shutdown(avd); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_RUN_CTRL), + (u32)(AVD_RUN_CTRL_UNK_STOP)); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_FLAG0_CLR), 1U); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX_IRQ_ENABLE), 0U); + } +} + +static void avd_pm_recovery_dart_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct iommu_domain domain = {}; + + /* A stopped device cannot be reset through powered-off registers. */ + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); + avd->pm.dart_active = true; + avd->domain = &domain; + /* No empty domain means no valid DART flush: never reboot anyway. */ + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -ENODEV); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); + KUNIT_EXPECT_EQ(test, fixture->boot_step, 0U); +} + +static void avd_pm_recovery_boot_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + avd->pm.dart_active = true; + fixture->boot_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 2U); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + fixture->boot_error = 0; + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), 0); + KUNIT_EXPECT_EQ(test, avd->fw_generation, 1ULL); + /* Recovery must fail old slots before core reopens the host IRQ. */ + KUNIT_EXPECT_FALSE(test, avd->pm.irq_enabled); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); +} + +static void avd_pm_parked_resume_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + /* Autosuspend/resume must not boot or reopen a resting/retired engine. */ + avd->pm.dma_parked = true; + avd->rest_until = jiffies + msecs_to_jiffies(60000); + KUNIT_EXPECT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_EQ(test, fixture->boot_step, 0U); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + KUNIT_EXPECT_FALSE(test, avd->pm.irq_enabled); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), 0); + KUNIT_EXPECT_TRUE(test, avd->pm.dma_parked); + avd->rest_until = 0; + avd->faulted = true; + KUNIT_EXPECT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_EQ(test, fixture->boot_step, 0U); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + KUNIT_EXPECT_FALSE(test, avd->pm.irq_enabled); +} + +static void avd_pm_test_exit(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + + if (fixture->queue_initialized) + cancel_delayed_work_sync(&fixture->avd.queue_work); +} + +static struct kunit_case avd_pm_test_cases[] = { + KUNIT_CASE(avd_pm_suspend_gate_test), + KUNIT_CASE(avd_pm_resume_order_test), + KUNIT_CASE(avd_pm_resume_failure_test), + KUNIT_CASE(avd_pm_suspend_failure_test), + KUNIT_CASE(avd_pm_deferred_unmap_test), + KUNIT_CASE(avd_pm_real_queue_resume_test), + KUNIT_CASE(avd_pm_real_queue_gate_test), + KUNIT_CASE(avd_pm_open_boot_replay_test), + KUNIT_CASE(avd_pm_recovery_dart_gate_test), + KUNIT_CASE(avd_pm_recovery_boot_test), + KUNIT_CASE(avd_pm_parked_resume_test), + {} +}; + +static struct kunit_suite avd_pm_test_suite = { + .name = "apple-avd-pm", + .init = avd_pm_test_init, + .exit = avd_pm_test_exit, + .test_cases = avd_pm_test_cases, +}; + +kunit_test_suite(avd_pm_test_suite); diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c new file mode 100644 index 0000000000000..01b9069e113ab --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include +#include +#include + +#include "avd.h" + +/* [C] Keep Linux's existing default until the idle-timer capture is available. */ +#define AVD_AUTOSUSPEND_MS 100 +#define AVD_DRAIN_TIMEOUT_MS 5000 +#define AVD_PIODMA_SRAM_BASE 0x24 + +void avd_pm_enable_irq(struct avd_dev *avd) +{ + if (avd->pm.irq >= 0 && !avd->pm.irq_enabled) { + mutex_lock(&avd->pm.hw_lock); + avd->pm.hw_on = true; + mutex_unlock(&avd->pm.hw_lock); + enable_irq(avd->pm.irq); + avd->pm.irq_enabled = true; + } +} + +/* + * Callers may hold queue_mutex, which the threaded handler takes first, so + * never wait for the handler here (disable_irq() would deadlock). Instead + * close hardware access under hw_lock: a handler already touching the + * decoder finishes before this returns, and any later one sees hw_on clear + * or a new irq_gen and leaves the hardware alone. + */ +static void avd_pm_disable_irq(struct avd_dev *avd) +{ + if (avd->pm.irq >= 0 && avd->pm.irq_enabled) { + mutex_lock(&avd->pm.hw_lock); + avd->pm.hw_on = false; + avd->pm.irq_gen++; + mutex_unlock(&avd->pm.hw_lock); + disable_irq_nosync(avd->pm.irq); + avd->pm.irq_enabled = false; + } +} + +/* Run the backend's interrupt handler only against a live firmware instance. */ +irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen) +{ + irqreturn_t ret = IRQ_HANDLED; + + /* Caller holds queue_mutex; hw_lock holders never wait for it. */ + mutex_lock(&avd->pm.hw_lock); + if (avd->pm.hw_on && gen == avd->pm.irq_gen) + ret = avd->backend->irq(avd); + mutex_unlock(&avd->pm.hw_lock); + return ret; +} + +struct avd_dma_release { + struct avd_deferred_unmap unmap; + void *cpu; + dma_addr_t addr; + size_t size; +}; + +static void avd_dma_unmap(struct avd_dev *avd, + struct avd_deferred_unmap *unmap) +{ + struct avd_dma_release *release; + + release = container_of(unmap, struct avd_dma_release, unmap); + dma_free_coherent(avd->dev, release->size, release->cpu, release->addr); + kfree(release); +} + +int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) +{ + struct avd_dma_release *release; + + avd_buf_free(avd, buf); + if (!size) + return -ENOMEM; + + /* Allocate the release record now: freeing must never need to allocate. */ + release = kzalloc_obj(*release); + if (!release) + return -ENOMEM; + INIT_LIST_HEAD(&release->unmap.node); + release->unmap.unmap = avd_dma_unmap; + release->size = size; + release->cpu = dma_alloc_coherent(avd->dev, size, &release->addr, GFP_KERNEL); + if (!release->cpu) { + kfree(release); + return -ENOMEM; + } + + /* Payload length, not an allocation rounding, is used by codec packers. */ + buf->cpu = release->cpu; + buf->addr = release->addr; + buf->size = size; + buf->release = release; + return 0; +} + +void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) +{ + if (buf->release) + avd_pm_unmap(avd, &buf->release->unmap); + /* The record outlives the context when unmapping has to be deferred. */ + memset(buf, 0, sizeof(*buf)); +} + +bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining) +{ + unsigned int i; + + if (draining) + return false; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (frames[i].state != AVD_FRAME_FREE) + return false; + + return true; +} + +static bool avd_pm_queue_idle(struct avd_dev *avd) +{ + unsigned long flags; + bool idle; + + spin_lock_irqsave(&avd->queue_lock, flags); + idle = avd_pm_can_suspend(avd->frames, avd->pm.draining); + spin_unlock_irqrestore(&avd->queue_lock, flags); + return idle; +} + +void avd_pm_queue_changed(struct avd_dev *avd) +{ + wake_up_all(&avd->pm.idle_wait); +} + +void avd_pm_drain_begin(struct avd_dev *avd) +{ + unsigned long flags; + + spin_lock_irqsave(&avd->queue_lock, flags); + avd->pm.draining++; + spin_unlock_irqrestore(&avd->queue_lock, flags); +} + +void avd_pm_drain_end(struct avd_dev *avd) +{ + unsigned long flags; + + spin_lock_irqsave(&avd->queue_lock, flags); + avd->pm.draining--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); +} + +static void avd_pm_resubmit(struct avd_dev *avd) +{ + /* queue_mutex may be held by the caller of pm_runtime_resume_and_get(). */ + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); +} + +static int avd_pm_queue_drain(struct avd_dev *avd) +{ + avd_pm_resubmit(avd); + if (!wait_event_timeout(avd->pm.idle_wait, avd_pm_queue_idle(avd), + msecs_to_jiffies(AVD_DRAIN_TIMEOUT_MS))) + return -EBUSY; + return 0; +} + +const struct avd_queue_pm_ops avd_pm_queue_ops = { + .idle = avd_pm_queue_idle, + .drain = avd_pm_queue_drain, + .resubmit = avd_pm_resubmit, +}; + +void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, + const struct avd_queue_pm_ops *queue) +{ + mutex_init(&avd->pm.lock); + INIT_LIST_HEAD(&avd->pm.deferred_unmaps); + init_waitqueue_head(&avd->pm.idle_wait); + avd->pm.dart_active = false; + avd->pm.irq = -1; + avd->pm.irq_enabled = false; + mutex_init(&avd->pm.hw_lock); + avd->pm.hw_on = false; + avd->pm.irq_gen = 0; + avd->backend = backend; + avd->pm.queue = queue; +} + +void avd_pm_enable(struct avd_dev *avd) +{ + pm_runtime_set_autosuspend_delay(avd->dev, AVD_AUTOSUSPEND_MS); + pm_runtime_use_autosuspend(avd->dev); + pm_runtime_enable(avd->dev); +} + +int avd_pm_map_piodma(struct avd_dev *avd) +{ + struct resource *piodma, *sram; + + piodma = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "piodma"); + /* Old DTs support only the legacy transport; never invent a resource. */ + if (!piodma) + return 0; + sram = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "sram"); + if (!sram || !IS_ALIGNED(sram->start, 16) || + (sram->start >> 4) > U32_MAX || + resource_size(piodma) < AVD_PIODMA_SRAM_BASE + sizeof(u32)) + return -EINVAL; + avd->pm.piodma = devm_ioremap_resource(avd->dev, piodma); + if (IS_ERR(avd->pm.piodma)) + return PTR_ERR(avd->pm.piodma); + avd->pm.sram_phys = sram->start; + if (of_count_phandle_with_args(avd->dev->of_node, "iommus", "#iommu-cells") < 2) + dev_warn(avd->dev, "PIODMA transport requires decode and DMA IOMMU streams\n"); + return 0; +} + +void avd_pm_prepare_piodma(struct avd_dev *avd) +{ + /* Public CM3 boot ABI: replay the SRAM base after every power loss/reset. */ + if (avd->pm.piodma) + writel((u32)(avd->pm.sram_phys >> 4), + avd->pm.piodma + AVD_PIODMA_SRAM_BASE); +} + +void avd_pm_disable(struct avd_dev *avd) +{ + int ret; + + /* Power suppliers before flushing frees queued by cleanup while idle. */ + ret = pm_runtime_resume_and_get(avd->dev); + if (ret >= 0) + ret = pm_runtime_put_sync_suspend(avd->dev); + if (ret < 0) + dev_warn(avd->dev, "final power transition failed: %d\n", ret); + pm_runtime_disable(avd->dev); + /* Also close the host IRQ if the final hardware transition failed. */ + avd_pm_disable_irq(avd); + /* No queue_mutex is held here: let a woken handler thread finish. */ + if (avd->pm.irq >= 0) + synchronize_irq(avd->pm.irq); + pm_runtime_dont_use_autosuspend(avd->dev); + /* Resume may have kicked the already-stopped queue. */ + cancel_delayed_work_sync(&avd->queue_work); + WARN_ON(!list_empty(&avd->pm.deferred_unmaps)); +} + +/* pm.lock held; release records own their storage and may free themselves. */ +static void avd_pm_drain_unmaps(struct avd_dev *avd) +{ + struct avd_deferred_unmap *unmap, *next; + + list_for_each_entry_safe(unmap, next, &avd->pm.deferred_unmaps, node) { + list_del_init(&unmap->node); + unmap->unmap(avd, unmap); + } +} + +void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap) +{ + mutex_lock(&avd->pm.lock); + if (avd->pm.dart_active) + unmap->unmap(avd, unmap); + else + list_add_tail(&unmap->node, &avd->pm.deferred_unmaps); + mutex_unlock(&avd->pm.lock); +} + +int avd_pm_backend_suspend(struct avd_dev *avd) +{ + avd->backend->shutdown(avd); + return 0; +} + +int avd_pm_backend_resume(struct avd_dev *avd) +{ + /* Includes transport setup, ITCM verification/load and a new generation. */ + return avd_backend_boot(avd); +} + +/* Caller holds a runtime PM reference and queue_mutex. */ +int avd_pm_park(struct avd_dev *avd) +{ + int ret; + + if (!avd->pm.dart_active) + return -EHOSTDOWN; + avd_pm_disable_irq(avd); + dev_warn(avd->dev, "recover: stop firmware\n"); + avd->backend->shutdown(avd); + if (avd->domain && !avd->pm.dma_parked) { + if (!avd->empty_domain) + return -ENODEV; + dev_warn(avd->dev, "recover: block DMA\n"); + ret = iommu_attach_device(avd->empty_domain, avd->dev); + if (ret) { + dev_err(avd->dev, "recover: DART empty attach failed: %d\n", ret); + return ret; + } + avd->pm.dma_parked = true; + /* Outstanding transfers drain into faults before any reset/free. */ + usleep_range(5000, 6000); + } + return 0; +} + +int avd_pm_recover(struct avd_dev *avd) +{ + bool suspended; + int ret; + + if (!avd->pm.dart_active) { + dev_err(avd->dev, "recover: decoder DART inactive, not touching it\n"); + return -EHOSTDOWN; + } + /* The PM KUnit fixture has no struct device. */ + suspended = avd->dev && pm_runtime_suspended(avd->dev); + dev_warn(avd->dev, "recover: start (runtime %s, irq %s)\n", + suspended ? "suspended" : "active", + avd->pm.irq_enabled ? "on" : "off"); + if (WARN_ONCE(suspended, + "avd: recovery with the decoder powered off\n")) + return -EHOSTDOWN; + ret = avd_pm_park(avd); + if (ret) + return ret; + dev_warn(avd->dev, "recover: reset\n"); + ret = reset_control_reset(avd->rstc); + if (ret) { + /* A failed reset must never restore the decoder's DMA mappings. */ + dev_err(avd->dev, "recover: reset failed: %d\n", ret); + return ret; + } + if (avd->pm.dma_parked) { + dev_warn(avd->dev, "recover: restore DMA\n"); + iommu_detach_device(avd->empty_domain, avd->dev); + if (iommu_get_domain_for_dev(avd->dev) != avd->domain) { + dev_err(avd->dev, "recover: DART domain not restored\n"); + return -EIO; + } + avd->pm.dma_parked = false; + } + dev_warn(avd->dev, "recover: boot firmware\n"); + ret = avd_pm_backend_resume(avd); + if (ret) { + dev_err(avd->dev, "recover: boot failed: %d\n", ret); + /* Boot may have started the engine: park it again before retiring. */ + avd_pm_park(avd); + return ret; + } + dev_warn(avd->dev, "recover: done\n"); + return 0; +} + +int avd_pm_power_on(struct avd_dev *avd) +{ + const struct avd_backend_ops *backend = avd->backend; + int ret; + + /* genpd and the IOMMU suppliers resume before this runtime callback. */ + mutex_lock(&avd->pm.lock); + avd->pm.dart_active = true; + avd_pm_drain_unmaps(avd); + mutex_unlock(&avd->pm.lock); + + /* Rest/retirement survives autosuspend: never boot with DMA parked. */ + if (avd->faulted || avd->rest_until || avd->pm.dma_parked) + return 0; + + /* Do not hold pm.lock: boot may allocate or release DMA buffers. */ + ret = backend->resume ? backend->resume(avd) : avd_pm_backend_resume(avd); + if (ret) { + avd->backend->shutdown(avd); + mutex_lock(&avd->pm.lock); + avd->pm.dart_active = false; + mutex_unlock(&avd->pm.lock); + return ret; + } + avd_pm_enable_irq(avd); + + if (avd->pm.queue->resubmit) + avd->pm.queue->resubmit(avd); + + return 0; +} + +int avd_pm_power_off(struct avd_dev *avd) +{ + const struct avd_backend_ops *backend = avd->backend; + int ret; + + /* PM references cover reservation through retirement, including drain. */ + if (!avd->pm.queue->idle(avd)) + return -EBUSY; + + if (backend->suspend && !avd->pm.dma_parked) { + ret = backend->suspend(avd); + if (ret) + return ret; + } else { + backend->shutdown(avd); + } + avd_pm_disable_irq(avd); + + /* No more firmware DMA; suppliers stay on until this callback returns. */ + mutex_lock(&avd->pm.lock); + avd_pm_drain_unmaps(avd); + avd->pm.dart_active = false; + mutex_unlock(&avd->pm.lock); + return 0; +} + +static int avd_runtime_resume(struct device *dev) +{ + return avd_pm_power_on(dev_get_drvdata(dev)); +} + +static int avd_runtime_suspend(struct device *dev) +{ + return avd_pm_power_off(dev_get_drvdata(dev)); +} + +static int avd_system_suspend(struct device *dev) +{ + struct avd_dev *avd = dev_get_drvdata(dev); + int ret; + + v4l2_m2m_suspend(avd->m2m_dev); + ret = avd->pm.queue->drain(avd); + if (!ret) + ret = pm_runtime_force_suspend(dev); + if (ret) + v4l2_m2m_resume(avd->m2m_dev); + return ret; +} + +static int avd_system_resume(struct device *dev) +{ + struct avd_dev *avd = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + v4l2_m2m_resume(avd->m2m_dev); + return 0; +} + +const struct dev_pm_ops avd_pm_ops = { + SYSTEM_SLEEP_PM_OPS(avd_system_suspend, avd_system_resume) + RUNTIME_PM_OPS(avd_runtime_suspend, avd_runtime_resume, NULL) +}; diff --git a/drivers/media/platform/apple/avd/avd-pm.h b/drivers/media/platform/apple/avd/avd-pm.h new file mode 100644 index 0000000000000..10e3a0ffbe936 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_PM_H_ +#define AVD_PM_H_ + +#include +#include +#include + +#include "avd-queue.h" + +struct avd_dma_release; + +struct avd_queue_pm_ops { + /* Snapshot under the queue lock: all FREE, no context draining. */ + bool (*idle)(struct avd_dev *avd); + /* Scheduling is paused; finish/retire all accepted frames. May sleep. */ + int (*drain)(struct avd_dev *avd); + /* Kick prepared RESERVED entries only, respecting dependencies. */ + void (*resubmit)(struct avd_dev *avd); +}; + +struct avd_deferred_unmap { + struct list_head node; + void (*unmap)(struct avd_dev *avd, struct avd_deferred_unmap *unmap); +}; + +struct avd_pm { + /* Serializes DMA releases against DART activation and firmware boot. */ + struct mutex lock; + struct list_head deferred_unmaps; + bool dart_active; + /* Empty DART domain remains attached while resting or retired. */ + bool dma_parked; + int irq; + bool irq_enabled; + /* + * hw_lock guards hw_on and irq_gen. The threaded handler touches the + * decoder only under hw_lock with hw_on set and an unchanged irq_gen, + * so turning the IRQ off never has to wait for the handler thread + * (which may itself be waiting for queue_mutex). + */ + /* A mutex: decoder MMIO is never done with interrupts off. */ + struct mutex hw_lock; + bool hw_on; + unsigned int irq_gen; + void __iomem *piodma; + phys_addr_t sram_phys; + wait_queue_head_t idle_wait; + /* Protected by queue_lock; counts drain operations, not stopped streams. */ + unsigned int draining; + const struct avd_queue_pm_ops *queue; +}; + +/* The caller holds the queue lock, including the draining-context count. */ +bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining); + +void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, + const struct avd_queue_pm_ops *queue); +void avd_pm_enable(struct avd_dev *avd); +int avd_pm_map_piodma(struct avd_dev *avd); +void avd_pm_prepare_piodma(struct avd_dev *avd); +/* After queue cleanup: release deferred DMA mappings before the final off. */ +void avd_pm_disable(struct avd_dev *avd); +/* Called in process context, after the last frame has retired its mapping. */ +void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap); +int avd_pm_power_on(struct avd_dev *avd); +int avd_pm_power_off(struct avd_dev *avd); +/* Caller holds a PM reference and queue_mutex; leaves the host IRQ closed. */ +int avd_pm_recover(struct avd_dev *avd); +/* Stop firmware and leave DMA blocked; same caller/IRQ contract as recover. */ +int avd_pm_park(struct avd_dev *avd); +/* After recovery has invalidated every previously firmware-owned slot. */ +void avd_pm_enable_irq(struct avd_dev *avd); +irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen); +void avd_pm_drain_begin(struct avd_dev *avd); +void avd_pm_drain_end(struct avd_dev *avd); +void avd_pm_queue_changed(struct avd_dev *avd); +extern const struct avd_queue_pm_ops avd_pm_queue_ops; + +/* Backends may share these when suspend loses all firmware context. */ +int avd_pm_backend_suspend(struct avd_dev *avd); +int avd_pm_backend_resume(struct avd_dev *avd); +extern const struct dev_pm_ops avd_pm_ops; + +#endif /* AVD_PM_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-pm.kunitconfig b/drivers/media/platform/apple/avd/avd-pm.kunitconfig new file mode 100644 index 0000000000000..4d9b37f71ff41 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.kunitconfig @@ -0,0 +1,14 @@ +CONFIG_KUNIT=y +CONFIG_COMPILE_TEST=y +CONFIG_OF=y +CONFIG_PM=y +CONFIG_SUSPEND=y +CONFIG_PM_SLEEP=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_WERROR=y diff --git a/drivers/media/platform/apple/avd/avd-protocol.rst b/drivers/media/platform/apple/avd/avd-protocol.rst new file mode 100644 index 0000000000000..ed75e333dd739 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-protocol.rst @@ -0,0 +1,100 @@ +AVD queued transport implementation status +========================================== + +The pool owns sixteen independently tagged slots. A source/destination pair +and a runtime-PM reference remain owned by the slot until in-order retirement +within its context. A duplicate or obsolete tag cannot complete a reused +slot. Inbox-full leaves a prepared slot reserved and retries without losing +its buffers. Timeout or streamoff resets the whole engine before failing all +slots affected by that reset. Slot deadlines survive inbox retries. + +The first commit adding avd-queue.h is the shared interface. Later additions +only extend it. Codec callbacks own codec-private state and must release that +state from retire(), including when prepare() failed part way through. + +Open firmware compatibility +-------------------------- + +The deployed, untagged open-firmware protocol has a one-frame limit. Existing +codec run() callbacks build a DMA-visible per-slot program; only the backend +writes those instructions to the VP FIFO. That compatibility backend retains +the old host-side PP kick. A native frame_ops program must use the queued +avd-fw transport before it can replace the compatibility path; this backend +returns EOPNOTSUPP for native programs rather than guessing their wire format. + +Partial HOLD_CAPTURE slice requests use buf_done_and_job_finish(), returning +the copied OUTPUT slice while CAPTURE remains held on the ready list. Final +slices use job_finish_detach(): it atomically transfers both ready buffers to +the frame slot and releases m2m dispatch without completing either buffer. +The next picture can prepare while the decoder still owns the earlier frame. +The slot returns those buffers only after its tagged completion or quiesced +recovery. The queue capability flags are never toggled around job_finish(). +This requires the matching V4L2 m2m detach API; an external-module deployment +must install the matching m2m module as well as apple-avd. + +Apple Borage transport +---------------------- + +Only T6040 has Borage match data. The Apple backend contains the 64-byte +little-endian connection configuration, session command builders, decode +prefix, eight-entry inbox publishing, a 64-entry completion shadow and bounded +20-word result parsing. Type 7 is VP/parsing completion and never releases a +buffer. The notification address rule has one definition, +AVD_APPLE_NOTIFY_BASE, so a capture can correct it. + +Boot checks a nonzero ITCM span and a nonzero/non-all-ones version word before +accepting a preloaded image. A separately installed firmware image can restore +ITCM after power loss. The last image word is zero padded without reading +past the firmware object. No Apple image is embedded in this source tree. + +Activation remains disabled in match data: the supplied clean specification +has no ITCM register offsets, link programming sequence, result-area address, +result field layout or final-completion value. It would be unsafe to invent +these or reuse the different open-firmware mailbox. The closest executable +path is the open compatibility backend. The Apple register description and +reply layout must be completed together from clean facts; setting verified +alone is insufficient. The DT must then describe the complete Apple aperture +as an owned named resource before probe can map it. + +Unresolved field ledger +----------------------- + +This is the single ledger for fields marked [C] in the common transport. No +semantic names have been assigned to the unknown bits. Codec-specific fields +remain in their codec packers' ledgers. + +============================== ================================================ +Field Source and required evidence +============================== ================================================ +Config +00/+08/+10/+18/+20/ SPEC 3.2: exact values, meanings unproved; +28/+2c/+30/+38 capture connection construction and LinkUp +ITCM start/end/clear/enable SPEC 3.1: operations, offsets/values absent; + clean per-SoC register ledger required +IMEM version-word offset SPEC 3.1: display format proved, offset absent +Mailbox link register writes SPEC 3.2: aperture 1, state 4, LinkUp proved; + offsets and bit operations absent +Mailbox ready register SPEC 3.1: status 1, poll 10 us; offset absent +Mailbox notification word SPEC 4.2; expected inbox-entry address from MIT + aura trace_avd.py V3 precedent; capture Borage +Inbox consumption/full rule SPEC 4.2: host positions modulo eight; + hardware consumer register/doorbell absent +Result-area base/indexing SPEC 4.3: 64 entries, 20 words; base and mailbox + notification interpretation absent +Reply type/client/frame/status SPEC 4.3: correlation proved, word layout absent; + synthetic KUnit layout is not a hardware fact +Final reply type/status values SPEC 4.3: only VP done = 7 proved; capture final + completion and error replies before activation +Decode +00 bit 9 SPEC 5.2: boolean argument, meaning unproved; + left zero until the producer is established +Decode +50..+5f SPEC 5: Borage extension, nonzero use unproved; + initialized to zero before codec packing +Commands 3/4 and other values SPEC 5.1: unproved; never sent +PIODMA mapping stream binding SPEC 7: three mapper roles proved, Linux binding + and per-domain fp/program DVA ownership pending +Retention and clock floors SPEC 8: per-SoC capture required; no guessed knobs +============================== ================================================ + +Licensed implementation references: eiln/avd (MIT), avd_emu.py, and +https://github.com/aurora-silicon/aura/blob/main/proxyclient/hv/trace_avd.py +(MIT). The Borage facts above come from the supplied clean SPEC sections 3-5; +no binary or captured Apple command images are used. diff --git a/drivers/media/platform/apple/avd/avd-queue-internal.h b/drivers/media/platform/apple/avd/avd-queue-internal.h new file mode 100644 index 0000000000000..c5b9c06836992 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue-internal.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_QUEUE_INTERNAL_H_ +#define AVD_QUEUE_INTERNAL_H_ + +#include +#include +#include + +#include "avd-queue.h" + +/* Caller serializes the pool; tags are never recycled during device lifetime. */ +static inline int avd_slot_reserve(struct avd_frame *f, struct avd_ctx *ctx, + u64 seq) +{ + if (f->state != AVD_FRAME_FREE) + return -EBUSY; + if (!seq || seq > (U32_MAX >> 4)) + return -EOVERFLOW; + f->seq = seq; + f->tag = (seq << 4) | f->idx; + f->ctx = ctx; + f->state = AVD_FRAME_RESERVED; + f->dep = AVD_DEP_NONE; + f->status = 0; + f->vp_done = false; + f->vp_retired = false; + f->prog_words = 0; + f->codec_priv = NULL; + return 0; +} + +/* A full inbox owns nothing: keep the prepared slot, tag and deadline. */ +static inline void avd_slot_submit_result(struct avd_frame *f, int status) +{ + if (status == -EBUSY) { + f->state = AVD_FRAME_RESERVED; + } else if (status) { + f->status = status; + f->state = AVD_FRAME_DONE; + } +} + +static inline bool avd_slot_complete(struct avd_frame *f, u32 tag, int status) +{ + if (f->state != AVD_FRAME_SUBMITTED || f->tag != tag) + return false; + f->status = status; + f->state = AVD_FRAME_DONE; + return true; +} + +static inline bool avd_slot_first(const struct avd_frame *pool, + const struct avd_frame *f) +{ + unsigned int i; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (pool[i].state != AVD_FRAME_FREE && pool[i].ctx == f->ctx && + pool[i].seq < f->seq) + return false; + return true; +} + +static inline bool avd_slot_expired(const struct avd_frame *f, unsigned long now) +{ + return (f->state == AVD_FRAME_SUBMITTED || + f->state == AVD_FRAME_RESERVED) && time_after_eq(now, f->deadline); +} + +/* Posted frames may overlap, but never pass an earlier reserved frame. */ +static inline bool avd_slot_submit_first(const struct avd_frame *pool, + const struct avd_frame *f) +{ + unsigned int i; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (pool[i].state == AVD_FRAME_RESERVED && pool[i].ctx == f->ctx && + pool[i].seq < f->seq) + return false; + return true; +} + +static inline bool avd_inbox_full(unsigned int producer, unsigned int consumer) +{ + return ((producer + 1) & 7) == consumer; +} + +#endif /* AVD_QUEUE_INTERNAL_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-queue-test.c b/drivers/media/platform/apple/avd/avd-queue-test.c new file mode 100644 index 0000000000000..8d343b1b93399 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue-test.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "avd.h" +#include "avd-apple-protocol.h" +#include "avd-queue-internal.h" + +static void avd_slots_and_tags(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *ctx = (void *)test; + unsigned int i; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + pool[i].idx = i; + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[i], ctx, i + 1), 0); + KUNIT_EXPECT_EQ(test, pool[i].tag, ((i + 1) << 4) | i); + KUNIT_EXPECT_EQ(test, pool[i].state, AVD_FRAME_RESERVED); + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[i], ctx, 100), -EBUSY); + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag, 0)); + pool[i].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag + 16, 0)); + KUNIT_EXPECT_TRUE(test, avd_slot_complete(&pool[i], pool[i].tag, 0)); + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag, -EIO)); + } + pool[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[0], ctx, 0), -EOVERFLOW); + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[0], ctx, (u64)(U32_MAX >> 4) + 1), + -EOVERFLOW); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], ctx, 100), 0); + pool[0].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[0], 16, 0)); +} + +static void avd_retirement_order(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *a = (void *)test; + struct avd_ctx *b = (void *)&pool; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], a, 1), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[1], b, 2), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[2], a, 3), 0); + pool[0].state = AVD_FRAME_SUBMITTED; + pool[1].state = AVD_FRAME_SUBMITTED; + pool[2].state = AVD_FRAME_SUBMITTED; + KUNIT_ASSERT_TRUE(test, avd_slot_complete(&pool[2], pool[2].tag, 0)); + KUNIT_EXPECT_FALSE(test, avd_slot_first(pool, &pool[2])); + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[1])); + KUNIT_ASSERT_TRUE(test, avd_slot_complete(&pool[0], pool[0].tag, -ETIMEDOUT)); + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[0])); + pool[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[2])); + KUNIT_EXPECT_EQ(test, pool[0].status, -ETIMEDOUT); +} + +static void avd_watchdog_and_retry(struct kunit *test) +{ + struct avd_frame f = { .state = AVD_FRAME_SUBMITTED, .deadline = ULONG_MAX - 2 }; + unsigned int producer, consumer; + u32 tag = 0x124; + unsigned long deadline = f.deadline; + + f.tag = tag; + avd_slot_submit_result(&f, -EBUSY); + KUNIT_EXPECT_EQ(test, f.state, AVD_FRAME_RESERVED); + KUNIT_EXPECT_EQ(test, f.tag, tag); + KUNIT_EXPECT_EQ(test, f.deadline, deadline); + f.state = AVD_FRAME_SUBMITTED; + avd_slot_submit_result(&f, 0); + KUNIT_EXPECT_EQ(test, f.state, AVD_FRAME_SUBMITTED); + + KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, ULONG_MAX - 3)); + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, ULONG_MAX - 2)); + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, 1)); + f.state = AVD_FRAME_DONE; + KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, 1)); + f.state = AVD_FRAME_RESERVED; + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, 1)); + for (producer = 0; producer < 8; producer++) { + for (consumer = 0; consumer < 8; consumer++) + KUNIT_EXPECT_EQ(test, avd_inbox_full(producer, consumer), + ((producer + 1) & 7) == consumer); + consumer = (producer + 1) & 7; + KUNIT_EXPECT_TRUE(test, avd_inbox_full(producer, consumer)); + /* A consumer advance reopens the same pending inbox slot. */ + consumer = (consumer + 1) & 7; + KUNIT_EXPECT_FALSE(test, avd_inbox_full(producer, consumer)); + KUNIT_EXPECT_EQ(test, avd_apple_notification(producer), + 0x01482cf8U + 0x60 * producer); + } +} + +static void avd_completion_parser(struct kunit *test) +{ + /* Synthetic layout for parser testing; never a claimed Borage wire ABI. */ + struct avd_apple_reply_layout layout = { + .type_word = 0, .type_mask = 0x1f, + .client_word = 1, .client_shift = 16, .client_mask = 0x3ff, + .frame_word = 3, .status_word = 4, .done_type = 8, .verified = true, + }; + struct avd_apple_reply reply; + u32 record[AVD_APPLE_RESULT_WORDS] = { 7, 5 << 16, 0, 0x12340 }; + + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 19, &layout, &reply), -EMSGSIZE); + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EINPROGRESS); + record[0] = 8; + KUNIT_ASSERT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), 0); + KUNIT_EXPECT_EQ(test, reply.client, 5U); + KUNIT_EXPECT_EQ(test, reply.frame, 0x12340U); + KUNIT_EXPECT_EQ(test, reply.status, 0); + record[4] = 1; + KUNIT_ASSERT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), 0); + KUNIT_EXPECT_EQ(test, reply.status, -EIO); + record[0] = 3; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -ENOMSG); + layout.frame_word = 20; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EINVAL); + layout.verified = false; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EOPNOTSUPP); +} + +static void avd_command_builders(struct kunit *test) +{ + struct avd_frame f = { .idx = 15, .frame_num = 0x1234 }; + u32 cmd[24]; + const u64 config[] = { + 0, 0x01558000, 0x01558a00, 0x01558e00, 0, + 0x000000040004000aULL, 0, 0x01100008, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(config); i++) + KUNIT_EXPECT_EQ(test, get_unaligned_le64(avd_borage_config + i * 8), config[i]); + KUNIT_ASSERT_EQ(test, avd_apple_decode_prefix(&f, 1023, 15, 3), 0); + KUNIT_EXPECT_EQ(test, f.cmd[0], 1U | (15U << 10) | (1023U << 22)); + KUNIT_EXPECT_EQ(test, f.cmd[1], 15U | (3U << 16)); + KUNIT_EXPECT_EQ(test, f.cmd[3], 0x1234U); + KUNIT_EXPECT_EQ(test, f.cmd[23], 0U); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 1024, 0, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 0, 16, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 0, 0, 4), -EINVAL); + KUNIT_ASSERT_EQ(test, avd_apple_session_command(cmd, 2, 6, 17, 42), 0); + KUNIT_EXPECT_EQ(test, cmd[0], 6U | (2U << 16)); + KUNIT_EXPECT_EQ(test, cmd[1], 17U); + KUNIT_EXPECT_EQ(test, cmd[2], 42U); + KUNIT_EXPECT_EQ(test, cmd[23], 0U); + KUNIT_EXPECT_EQ(test, avd_apple_session_command(cmd, 2, 3, 0, 0), -EINVAL); +} + +static unsigned int avd_test_max_inflight(struct avd_dev *avd) +{ + return AVD_MAX_FRAMES; +} + +static void avd_dispatch_reservation(struct kunit *test) +{ + static const struct avd_backend_ops backend = { + .max_inflight = avd_test_max_inflight, + }; + struct avd_dev *avd = kunit_kzalloc(test, sizeof(*avd), GFP_KERNEL); + struct avd_ctx *ctx = kunit_kcalloc(test, 2, sizeof(*ctx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, avd); + KUNIT_ASSERT_NOT_NULL(test, ctx); + spin_lock_init(&avd->queue_lock); + avd->backend = &backend; + ctx[0].dev = avd; + ctx[1].dev = avd; + /* Every context with work is ready; capacity is checked at dispatch. */ + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[0]), 1); + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); + avd->inflight = AVD_MAX_FRAMES; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); + avd->inflight = 1; + avd->frames[0].state = AVD_FRAME_SUBMITTED; + avd->frames[0].ctx = &ctx[1]; + ctx[1].next_dep = AVD_DEP_PREV_DONE; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 0); + avd->frames[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); +} + +static void avd_submit_order(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *a = (void *)test, *b = (void *)&pool; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + pool[1].idx = 1; + pool[2].idx = 2; + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], a, 1), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[1], a, 2), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[2], b, 3), 0); + KUNIT_EXPECT_TRUE(test, avd_slot_submit_first(pool, &pool[0])); + KUNIT_EXPECT_FALSE(test, avd_slot_submit_first(pool, &pool[1])); + KUNIT_EXPECT_TRUE(test, avd_slot_submit_first(pool, &pool[2])); + pool[0].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_TRUE(test, avd_slot_submit_first(pool, &pool[1])); + avd_slot_submit_result(&pool[0], -EBUSY); + KUNIT_EXPECT_FALSE(test, avd_slot_submit_first(pool, &pool[1])); +} + +static struct kunit_case avd_queue_cases[] = { + KUNIT_CASE(avd_slots_and_tags), + KUNIT_CASE(avd_retirement_order), + KUNIT_CASE(avd_watchdog_and_retry), + KUNIT_CASE(avd_completion_parser), + KUNIT_CASE(avd_command_builders), + KUNIT_CASE(avd_dispatch_reservation), + KUNIT_CASE(avd_submit_order), + {} +}; + +static struct kunit_suite avd_queue_suite = { + .name = "apple-avd-queue", + .test_cases = avd_queue_cases, +}; + +kunit_test_suite(avd_queue_suite); diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c new file mode 100644 index 0000000000000..22e40b631a527 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -0,0 +1,707 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-queue-internal.h" + +#define AVD_FRAME_TIMEOUT msecs_to_jiffies(2000) +/* A stream is stopped at its first frame that fails in hardware. */ +#define AVD_CTX_MAX_HW_ERRORS 1 +/* Never reset the decoder more often than this. */ +#define AVD_RECOVER_GAP_MS 20 +/* A third recovery parks the decoder until AVD_STORM_WINDOW ends. */ +#define AVD_STORM_MAX 2 +#define AVD_STORM_WINDOW msecs_to_jiffies(60000) +#define AVD_RETRY_DELAY msecs_to_jiffies(1) +#define AVD_PROGRAM_SIZE SZ_2M + +/* PM test fixtures use backends without a transport. */ +static unsigned int avd_max_inflight(struct avd_dev *avd) +{ + return avd->backend->max_inflight ? avd->backend->max_inflight(avd) : 1; +} + +bool avd_backend_is_apple(const struct avd_dev *avd) +{ + return avd->backend == &avd_apple_backend; +} + +/* Only the CPU callback releases a probability dependency, never the IRQ. */ +static void avd_retire_vp(struct avd_dev *avd) +{ + unsigned int i; + + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + const struct avd_coded_fmt_ops *ops; + + if (f->state == AVD_FRAME_FREE || !f->vp_done || f->vp_retired) + continue; + ops = f->ctx->coded_fmt_desc->ops; + /* A reset/error must not publish partial symbol counts. */ + if (!f->status && ops && ops->vp_done) { + dma_rmb(); + ops->vp_done(f->ctx, f->dst); + } + f->vp_retired = true; + } +} + +/* queue_mutex excludes callbacks and context destruction from retirement. */ +static void avd_retire(struct avd_dev *avd) +{ + struct avd_frame *f; + unsigned long flags; + unsigned int i; + bool progress; + + do { + progress = false; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_ctx *ctx; + enum vb2_buffer_state state; + + f = &avd->frames[i]; + spin_lock_irqsave(&avd->queue_lock, flags); + if (f->state != AVD_FRAME_DONE || + !avd_slot_first(avd->frames, f)) { + spin_unlock_irqrestore(&avd->queue_lock, flags); + continue; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + ctx = f->ctx; + state = f->status ? VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE; + if (!f->status) { + ctx->hw_errors = 0; + } else if (++ctx->hw_errors >= AVD_CTX_MAX_HW_ERRORS && !ctx->broken) { + ctx->broken = true; + dev_err(avd->dev, "stream stopped after %u failed frames; restart streaming to retry\n", + ctx->hw_errors); + } + if (ctx->coded_fmt_desc->frame_ops) { + if (ctx->coded_fmt_desc->frame_ops->retire) + ctx->coded_fmt_desc->frame_ops->retire(ctx, f, state); + } else if (ctx->coded_fmt_desc->ops->done) { + ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, state); + } + if (ctx->coded_fmt_desc->frame_ops && f->src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, + &ctx->ctrl_hdl); + /* CAPTURE first: OUTPUT completion signals the media request. */ + if (f->scheduler_held) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, + ctx->fh.m2m_ctx, state); + } else { + v4l2_m2m_buf_done(f->dst, state); + v4l2_m2m_buf_done(f->src, state); + } + spin_lock_irqsave(&avd->queue_lock, flags); + f->ctx = NULL; + f->src = NULL; + f->dst = NULL; + f->state = AVD_FRAME_FREE; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + progress = true; + } + } while (progress); +} + +/* A reset invalidates every firmware-owned slot, including other clients. */ +static void avd_reset_queue(struct avd_dev *avd, int status) +{ + struct avd_ctx *ctx; + unsigned long flags; + unsigned int i; + int ret; + + /* + * Every decoder reset is a risk to the whole machine (three hard hangs + * on the J616s followed timeouts and resets), so recover sparingly: + * at least AVD_RECOVER_GAP_MS apart, at most AVD_STORM_MAX times in + * AVD_STORM_WINDOW, and never again after a recovery fails. Exhausting + * the budget parks DMA until the window ends; only a failed recovery + * retires the decoder until the module is reloaded. + */ + if (avd->faulted) { + ret = -ENODEV; + } else { + if (!avd->storm_count || + time_after(jiffies, avd->storm_start + AVD_STORM_WINDOW)) { + avd->storm_start = jiffies; + avd->storm_count = 0; + } + if (++avd->storm_count > AVD_STORM_MAX) { + ret = avd_pm_park(avd); + if (!ret) { + avd->rest_until = avd->storm_start + AVD_STORM_WINDOW; + dev_warn(avd->dev, "decoder resting after repeated resets, new frames fail for up to %u s\n", + jiffies_to_msecs(avd->rest_until - jiffies) / 1000 + 1); + ret = -EBUSY; + } + } else { + if (avd->last_recover && + time_before(jiffies, avd->last_recover + + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) + msleep(AVD_RECOVER_GAP_MS); + ret = avd_pm_recover(avd); + avd->last_recover = jiffies ?: 1; + } + if (ret && !avd->rest_until) { + WRITE_ONCE(avd->faulted, true); + dev_err(avd->dev, "decoder retired after a failed reset (%d): reload the module to use it again\n", + ret); + } + } + avd->compat_active = NULL; + list_for_each_entry(ctx, &avd->contexts, queue_node) + ctx->session_started = false; + spin_lock_irqsave(&avd->queue_lock, flags); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) + continue; + avd->frames[i].status = status; + avd->frames[i].state = AVD_FRAME_DONE; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (!ret) + avd_pm_enable_irq(avd); +} + +static int avd_send_frame(struct avd_dev *avd, struct avd_frame *f) +{ + unsigned long flags; + int ret; + + if (!avd_slot_submit_first(avd->frames, f)) + return -EBUSY; + if (f->dep == AVD_DEP_PREV_DONE && !avd_slot_first(avd->frames, f)) + return -EBUSY; + if ((!f->ctx->session_started || + f->ctx->session_generation != avd->fw_generation) && + avd->backend->session_start) { + ret = avd->backend->session_start(f->ctx); + if (ret) + return ret; + f->ctx->session_started = true; + f->ctx->session_generation = avd->fw_generation; + } + /* Publish ownership before the doorbell; IRQs may arrive in submit(). */ + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_SUBMITTED; + spin_unlock_irqrestore(&avd->queue_lock, flags); + dma_wmb(); + ret = avd->backend->submit(avd, f); + spin_lock_irqsave(&avd->queue_lock, flags); + avd_slot_submit_result(f, ret); + spin_unlock_irqrestore(&avd->queue_lock, flags); + return ret; +} + +static void avd_queue_worker(struct work_struct *work) +{ + struct avd_dev *avd = container_of(to_delayed_work(work), + struct avd_dev, queue_work); + struct avd_ctx *ctx; + unsigned long flags; + unsigned int i; + bool pending; + unsigned long delay = AVD_FRAME_TIMEOUT; + + mutex_lock(&avd->queue_mutex); + if (avd->stopping) + goto unlock; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + bool expired; + + spin_lock_irqsave(&avd->queue_lock, flags); + expired = avd_slot_expired(&avd->frames[i], jiffies); + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (expired) { + struct avd_frame *t = &avd->frames[i]; + + dev_err(avd->dev, "frame tag %#x timed out (%p4cc, state %d, %u in flight, %s)\n", + t->tag, t->ctx ? &t->ctx->coded_fmt_desc->fourcc : &(u32){0}, + t->state, avd->inflight, avd->backend->name); + avd_qfw_debug(avd, t, "timeout"); + avd_reset_queue(avd, -ETIMEDOUT); + break; + } + } + avd_retire_vp(avd); + avd_retire(avd); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + int ret; + + if (f->state != AVD_FRAME_RESERVED) + continue; + ret = avd_send_frame(avd, f); + if (ret && ret != -EBUSY) { + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + } + } + avd_retire(avd); + /* Restart a job that found no free slot. */ + spin_lock_irqsave(&avd->queue_lock, flags); + ctx = avd->inflight < avd_max_inflight(avd) ? avd->deferred_ctx : NULL; + if (ctx) + avd->deferred_ctx = NULL; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (ctx) + schedule_work(&ctx->submit_work); + /* Freed slots/dependencies must wake contexts whose job_ready was false. */ + list_for_each_entry(ctx, &avd->contexts, queue_node) + v4l2_m2m_try_schedule(ctx->fh.m2m_ctx); + spin_lock_irqsave(&avd->queue_lock, flags); + pending = avd->inflight != 0; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + unsigned long remaining; + + if (f->state == AVD_FRAME_FREE) + continue; + remaining = time_after(f->deadline, jiffies) ? f->deadline - jiffies : 0; + if (f->state == AVD_FRAME_RESERVED) + remaining = min(remaining, AVD_RETRY_DELAY); + delay = min(delay, remaining); + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (pending) + mod_delayed_work(system_dfl_wq, &avd->queue_work, delay); + unlock: + mutex_unlock(&avd->queue_mutex); +} + +void avd_frame_complete(struct avd_dev *avd, u32 tag, int status) +{ + struct avd_frame *f = &avd->frames[tag & (AVD_MAX_FRAMES - 1)]; + unsigned long flags; + bool accepted; + + spin_lock_irqsave(&avd->queue_lock, flags); + accepted = avd_slot_complete(f, tag, status); + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (accepted) + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); +} + +/* No buffer, FIFO or descriptor ownership is released by a VP event. */ +void avd_frame_vp_complete(struct avd_dev *avd, u32 tag) +{ + struct avd_frame *f = &avd->frames[tag & (AVD_MAX_FRAMES - 1)]; + unsigned long flags; + bool accepted; + + spin_lock_irqsave(&avd->queue_lock, flags); + accepted = f->state == AVD_FRAME_SUBMITTED && f->tag == tag && !f->vp_done; + if (accepted) + f->vp_done = true; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (accepted) + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); +} + +int avd_queue_ready(void *priv) +{ + struct avd_ctx *ctx = priv; + struct avd_dev *avd = ctx->dev; + unsigned long flags; + unsigned int i; + bool ready; + + spin_lock_irqsave(&avd->queue_lock, flags); + /* + * A retired decoder still schedules: submission fails + * those frames at once, so clients see errors instead of waiting. + */ + /* + * Slot capacity is not a readiness condition: every context with work + * enters the m2m job queue, which runs them in order, and a job that + * finds no free slot is deferred until one retires. Gating readiness on + * one reserved context let the first context in the list win every + * time and starved the others (multi-stream decodes timing out). + */ + ready = !READ_ONCE(ctx->draining) && !READ_ONCE(avd->stopping); + if (ready && ctx->next_dep == AVD_DEP_PREV_DONE) { + /* Legacy codecs may hold several pictures in separate banks. */ + unsigned int limit = ctx->coded_fmt_desc && !ctx->coded_fmt_desc->frame_ops ? + max(ctx->max_frames, 1U) : 1, held = 0; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state != AVD_FRAME_FREE && + avd->frames[i].ctx == ctx) + held++; + ready = held < limit; + } + if (ready && ctx->coded_fmt_desc && !ctx->coded_fmt_desc->frame_ops && + ctx->coded_fmt_desc->ops->ready) + ready = ctx->coded_fmt_desc->ops->ready(ctx); + spin_unlock_irqrestore(&avd->queue_lock, flags); + return ready; +} + +static void avd_queue_submit_work(struct work_struct *work) +{ + struct avd_ctx *ctx = container_of(work, struct avd_ctx, submit_work); + struct avd_dev *avd = ctx->dev; + const struct avd_codec_frame_ops *ops = ctx->coded_fmt_desc->frame_ops; + struct avd_frame *f = NULL; + unsigned long flags; + unsigned int i; + int ret; + + mutex_lock(&avd->queue_mutex); + /* A queued retry can outlive drain/abort finishing its m2m dispatch. */ + if (v4l2_m2m_get_curr_priv(avd->m2m_dev) != ctx) + goto unlock; + spin_lock_irqsave(&avd->queue_lock, flags); + if (avd->deferred_ctx == ctx) + avd->deferred_ctx = NULL; + if (avd->stopping || ctx->draining || READ_ONCE(avd->faulted) || ctx->broken || + (avd->rest_until && time_before(jiffies, avd->rest_until))) { + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + + spin_unlock_irqrestore(&avd->queue_lock, flags); + /* Complete the request's controls too, or its fd never signals. */ + if (src && src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + goto unlock; + } + if (avd->inflight < avd_max_inflight(avd)) { + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) { + f = &avd->frames[i]; + break; + } + } + } + if (!f) { + /* Keep the m2m job; the worker restarts it when a slot frees. */ + avd->deferred_ctx = ctx; + spin_unlock_irqrestore(&avd->queue_lock, flags); + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); + goto unlock; + } + ret = avd_slot_reserve(f, ctx, ++avd->frame_seq); + if (!ret) + avd->inflight++; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (WARN_ON(ret)) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + goto unlock; + } + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + goto fail_no_pm; + f->src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + f->dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + /* + * Partial slices retain CAPTURE through the ordinary m2m HOLD API. + * Final slices detach both buffers and release dispatch atomically; + * neither DMA-owned buffer is completed by that early handoff. + */ + f->scheduler_held = (ctx->fh.m2m_ctx->out_q_ctx.q.subsystem_flags & + VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF) && + (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF); + f->frame_num = f->tag; + f->deadline = jiffies + AVD_FRAME_TIMEOUT; + if (avd->rest_until) { + /* Reset while still parked; retain the recovery pacing at expiry. */ + if (avd->last_recover && + time_before(jiffies, avd->last_recover + + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) + msleep(AVD_RECOVER_GAP_MS); + ret = avd_pm_recover(avd); + avd->last_recover = jiffies ?: 1; + if (ret) { + WRITE_ONCE(avd->faulted, true); + dev_err(avd->dev, "decoder retired after a failed wake (%d)\n", ret); + goto fail; + } + avd->rest_until = 0; + avd->storm_count = 0; + avd_pm_enable_irq(avd); + } + memset(f->cmd, 0, sizeof(f->cmd)); + if (ops) { + if (!f->prog->cpu) { + ret = avd_buf_alloc(avd, f->prog, AVD_PROGRAM_SIZE); + if (ret) + goto fail; + } + /* NULL fp is also the codec-facing indication of an open program. */ + f->fp = NULL; + if (avd_backend_is_apple(avd)) { + struct avd_buf *fp = avd->frame_fp[f->idx]; + + if (!fp) { + fp = devm_kzalloc(avd->dev, sizeof(*fp), GFP_KERNEL); + avd->frame_fp[f->idx] = fp; + } + f->fp = fp; + if (!f->fp) { + ret = -ENOMEM; + goto fail; + } + if (!f->fp->cpu) { + ret = avd_buf_alloc(avd, f->fp, avd->variant->fp_size); + if (ret) + goto fail; + } + } + if (f->src->vb2_buf.req_obj.req) { + ret = v4l2_ctrl_request_setup(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + if (ret) + goto fail; + } + ret = ops->prepare(ctx, f); + } else { + avd->building = f; + avd->compat_held = false; + ret = ctx->coded_fmt_desc->ops->run(ctx); + avd->building = NULL; + } + if (ret) + goto fail; + ctx->next_dep = ops ? f->dep : AVD_DEP_PREV_DONE; + if ((ops && (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) || + (!ops && avd->compat_held)) { + if (ops && ops->retire) + ops->retire(ctx, f, VB2_BUF_STATE_DONE); + else if (!ops && ctx->coded_fmt_desc->ops->done) + ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, VB2_BUF_STATE_DONE); + if (ops && f->src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_FREE; + f->ctx = NULL; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_DONE); + goto unlock; + } + v4l2_m2m_buf_copy_metadata(f->src, f->dst); + ret = avd_send_frame(avd, f); + if (ret && ret != -EBUSY) { + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + } + /* Atomically detach both buffers to the slot and release dispatch. */ + if (!f->scheduler_held) + v4l2_m2m_job_finish_detach(avd->m2m_dev, ctx->fh.m2m_ctx); + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); + goto unlock; + fail: + /* Failed preparation still transfers its buffers before retirement. */ + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (!f->scheduler_held) + v4l2_m2m_job_finish_detach(avd->m2m_dev, ctx->fh.m2m_ctx); + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); + goto unlock; + fail_no_pm: + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_FREE; + f->ctx = NULL; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + unlock: + mutex_unlock(&avd->queue_mutex); +} + +void avd_queue_run(void *priv) +{ + struct avd_ctx *ctx = priv; + + /* try_schedule() may invoke device_run synchronously under queue_mutex. */ + schedule_work(&ctx->submit_work); +} + +/* True while any slot still belongs to @ctx; the caller holds no queue lock. */ +static bool avd_ctx_has_frames(struct avd_dev *avd, struct avd_ctx *ctx) +{ + unsigned long flags; + unsigned int i; + bool busy = false; + + spin_lock_irqsave(&avd->queue_lock, flags); + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) + busy = true; + spin_unlock_irqrestore(&avd->queue_lock, flags); + return busy; +} + +void avd_queue_drain(struct avd_ctx *ctx) +{ + struct avd_dev *avd = ctx->dev; + unsigned long flags; + unsigned int i; + bool finish_deferred = false; + int ret; + + /* Session stop/reset accesses registers even when there are no frames. */ + ret = pm_runtime_resume_and_get(avd->dev); + avd_pm_drain_begin(avd); + mutex_lock(&avd->queue_mutex); + spin_lock_irqsave(&avd->queue_lock, flags); + WRITE_ONCE(ctx->draining, true); + if (avd->deferred_ctx == ctx) { + avd->deferred_ctx = NULL; + finish_deferred = true; + } + /* Prepared but never handed to the decoder: nothing to wait for. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + if (f->ctx == ctx && f->state == AVD_FRAME_RESERVED) { + f->status = -ECANCELED; + f->state = AVD_FRAME_DONE; + } + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + /* A deferred job is still m2m's running job: end it so abort can wait. */ + if (finish_deferred) + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + avd_retire(avd); + + /* + * Frames the decoder already owns finish on their own, or the queue + * worker times them out and recovers. Closing one stream must not reset + * the decoder under every other stream's frames: that reset, with their + * DMA in flight, preceded the J616s hangs. Wait for this context's own + * frames instead, without the queue mutex so the worker can retire them. + */ + if (ret >= 0 && !avd->stopping && avd_ctx_has_frames(avd, ctx)) { + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); + mutex_unlock(&avd->queue_mutex); + wait_event_timeout(avd->pm.idle_wait, !avd_ctx_has_frames(avd, ctx), + AVD_FRAME_TIMEOUT + msecs_to_jiffies(1000)); + mutex_lock(&avd->queue_mutex); + avd_retire(avd); + } + /* Last resort (worker stopped or wedged): cancel through a reset. */ + for (i = 0; ret >= 0 && i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) { + dev_warn(avd->dev, "stream close: frames still owned by the decoder, resetting\n"); + if (avd->backend->abort) + avd->backend->abort(ctx); + avd_reset_queue(avd, -ECANCELED); + break; + } + } + avd_retire(avd); + if (ret >= 0 && ctx->session_started && avd->backend->session_stop && + ctx->session_generation == avd->fw_generation) + avd->backend->session_stop(ctx); + ctx->session_started = false; + if (!avd->stopping) + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); + mutex_unlock(&avd->queue_mutex); + avd_pm_drain_end(avd); + if (ret >= 0) { + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + } +} + +void avd_queue_abort(void *priv) +{ + avd_queue_drain(priv); +} + +void avd_queue_add_ctx(struct avd_ctx *ctx) +{ + struct avd_ctx *other; + u32 id; + + INIT_WORK(&ctx->submit_work, avd_queue_submit_work); + mutex_lock(&ctx->dev->queue_mutex); + /* Borage decode prefix has ten client bits. */ + for (id = 0; id < 1024; id++) { + bool used = false; + + list_for_each_entry(other, &ctx->dev->contexts, queue_node) + used |= other->client_id == id; + if (!used) + break; + } + ctx->client_id = id; + list_add_tail(&ctx->queue_node, &ctx->dev->contexts); + mutex_unlock(&ctx->dev->queue_mutex); +} + +void avd_queue_del_ctx(struct avd_ctx *ctx) +{ + avd_queue_drain(ctx); + mutex_lock(&ctx->dev->queue_mutex); + if (ctx->dev->compat_ctx == ctx) + ctx->dev->compat_ctx = NULL; + list_del_init(&ctx->queue_node); + mutex_unlock(&ctx->dev->queue_mutex); +} + +int avd_queue_init(struct avd_dev *avd) +{ + unsigned int i; + + spin_lock_init(&avd->queue_lock); + mutex_init(&avd->queue_mutex); + INIT_LIST_HEAD(&avd->contexts); + INIT_DELAYED_WORK(&avd->queue_work, avd_queue_worker); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + f->idx = i; + f->prog = devm_kzalloc(avd->dev, sizeof(*f->prog), GFP_KERNEL); + if (!f->prog) + return -ENOMEM; + } + return 0; +} + +void avd_queue_cleanup(struct avd_dev *avd) +{ + unsigned int i; + + mutex_lock(&avd->queue_mutex); + WRITE_ONCE(avd->stopping, true); + if (avd->inflight) { + avd_reset_queue(avd, -ESHUTDOWN); + avd_retire(avd); + } + mutex_unlock(&avd->queue_mutex); + cancel_delayed_work_sync(&avd->queue_work); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + avd_buf_free(avd, avd->frames[i].prog); + if (avd->frame_fp[i]) + avd_buf_free(avd, avd->frame_fp[i]); + } +} diff --git a/drivers/media/platform/apple/avd/avd-queue.h b/drivers/media/platform/apple/avd/avd-queue.h new file mode 100644 index 0000000000000..038c4f911313e --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_QUEUE_H_ +#define AVD_QUEUE_H_ + +#include +#include +#include + +struct avd_dev; +struct avd_ctx; +struct avd_buf; /* avd.h: { void *cpu; dma_addr_t addr; size_t size; } */ + +#define AVD_MAX_FRAMES 16 /* SPEC 4.1: 16 frame-param slots */ + +enum avd_frame_state { + AVD_FRAME_FREE = 0, /* SPEC: slot state 0 */ + AVD_FRAME_RESERVED, /* 1: codec is filling it on the CPU */ + AVD_FRAME_SUBMITTED, /* 3: owned by firmware */ + AVD_FRAME_DONE, /* completion seen, waiting for in-order retire */ +}; + +/* Codec says what must finish before this frame may be submitted. */ +enum avd_frame_dep { + AVD_DEP_NONE = 0, /* independent: may overlap the previous frame */ + AVD_DEP_PREV_DONE, /* previous frame of this ctx must be DONE first + * (VP9 backward adaptation, AV1 CDF update, ...) */ +}; + +struct avd_frame { + unsigned int idx; /* 0..AVD_MAX_FRAMES-1, also the fp slot */ + u32 tag; /* (seq << 4) | idx, echoed by completion */ + u64 seq; /* global submission order */ + enum avd_frame_state state; + enum avd_frame_dep dep; + int status; /* 0 or -EIO/-ETIMEDOUT from the backend */ + + struct avd_ctx *ctx; + struct vb2_v4l2_buffer *src; /* OUTPUT, held until retire */ + struct vb2_v4l2_buffer *dst; /* CAPTURE, held until retire */ + u32 frame_num; /* SPEC 5.2 +0x0c */ + + /* open backend: instruction program, DMA-visible, built by the codec */ + struct avd_buf *prog; /* per-slot, allocated by core */ + size_t prog_words; + + /* apple backend: frame-params slot (SPEC 0x1130d0 on Borage) */ + struct avd_buf *fp; /* per-slot, allocated by core, size from match data */ + u32 cmd[24]; /* 0x60 Borage command; core fills the 5.2 prefix, + * codec pack_cmd() fills the codec part */ + + void *codec_priv; /* per-frame codec state, owned by the codec */ + unsigned long deadline; /* jiffies, per-frame watchdog */ + /* True while the buffers still use m2m's held-CAPTURE completion path. */ + bool scheduler_held; + /* Tagged VP event and CPU codec callback, independent of PP retirement. */ + bool vp_done; + bool vp_retired; + /* VP to PP instruction FIFO for this frame (open backends). */ + dma_addr_t fifo_iova; +}; + +struct avd_backend_ops { + const char *name; + int (*probe)(struct avd_dev *avd); /* apertures, fw source, version */ + int (*boot)(struct avd_dev *avd); /* power-up: fw present/loaded, link up, ready */ + void (*shutdown)(struct avd_dev *avd); + unsigned int (*max_inflight)(struct avd_dev *avd); + int (*session_start)(struct avd_ctx *ctx); /* apple: cmd 0; open: may be NULL */ + void (*session_stop)(struct avd_ctx *ctx); /* apple: cmd 2 */ + /* Non-blocking. -EBUSY means inbox full: core retries on next completion. */ + int (*submit)(struct avd_dev *avd, struct avd_frame *f); + void (*abort)(struct avd_ctx *ctx); /* apple: cmd 5 */ + irqreturn_t (*irq)(struct avd_dev *avd); /* calls avd_frame_complete() */ + int (*suspend)(struct avd_dev *avd); + int (*resume)(struct avd_dev *avd); + int (*set_property)(struct avd_ctx *ctx, u32 property, u32 value); +}; + +/* Codec half. Replaces avd_coded_fmt_ops.run/submit/done. CPU only, no MMIO. */ +struct avd_codec_frame_ops { + /* Read the request's controls, set f->dep, fill f->prog (open backend) + * or f->fp (apple backend), allocate f->codec_priv if needed. */ + int (*prepare)(struct avd_ctx *ctx, struct avd_frame *f); + /* Apple backend: fill the codec part of f->cmd (SPEC 6.x). */ + int (*pack_cmd)(struct avd_ctx *ctx, struct avd_frame *f); + /* Called in submission order per ctx, after completion or failure. */ + void (*retire)(struct avd_ctx *ctx, struct avd_frame *f, + enum vb2_buffer_state state); +}; + +/* Core API used by backends and codecs. */ +void avd_frame_complete(struct avd_dev *avd, u32 tag, int status); /* any ctx, IRQ-safe */ +void avd_frame_vp_complete(struct avd_dev *avd, u32 tag); +bool avd_backend_is_apple(const struct avd_dev *avd); + +#endif /* AVD_QUEUE_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 23917fd62bdca..46f856eb1d314 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -890,15 +890,16 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) { struct avd_ctx *ctx = vb2_get_drv_priv(q); - /* A new stream gets a fresh start after a hardware failure. */ - if (!V4L2_TYPE_IS_CAPTURE(q->type)) - ctx->broken = false; const struct avd_coded_fmt_desc *desc; struct vb2_v4l2_buffer *vbuf; int ret; + WRITE_ONCE(ctx->draining, false); if (V4L2_TYPE_IS_CAPTURE(q->type)) return 0; + /* A new stream gets a fresh hardware error budget. */ + ctx->hw_errors = 0; + ctx->broken = false; desc = ctx->coded_fmt_desc; if (WARN_ON(!desc)) { @@ -921,7 +922,7 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) return ret; } -static void avd_queue_cleanup(struct vb2_queue *vq, u32 state) +static void avd_m2m_queue_cleanup(struct vb2_queue *vq, u32 state) { struct avd_ctx *ctx = vb2_get_drv_priv(vq); @@ -946,6 +947,7 @@ static void avd_stop_streaming(struct vb2_queue *q) { struct avd_ctx *ctx = vb2_get_drv_priv(q); + avd_queue_drain(ctx); if (V4L2_TYPE_IS_OUTPUT(q->type)) { const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; @@ -960,7 +962,7 @@ static void avd_stop_streaming(struct vb2_queue *q) ctx->job.segments = NULL; } - avd_queue_cleanup(q, VB2_BUF_STATE_ERROR); + avd_m2m_queue_cleanup(q, VB2_BUF_STATE_ERROR); } const struct vb2_ops avd_queue_ops = { @@ -974,36 +976,17 @@ const struct vb2_ops avd_queue_ops = { .stop_streaming = avd_stop_streaming, }; +/* Legacy multi-slice codecs finish held OUTPUT requests during prepare. */ void avd_job_finish_no_pm(struct avd_ctx *ctx, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; - unsigned long flags; - - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx == ctx) { - avd->job_state = AVD_JOB_IDLE; - avd->job_ctx = NULL; - avd->job_pending = false; - } - spin_unlock_irqrestore(&avd->job_lock, flags); - - if (ctx->coded_fmt_desc->ops->done) { - struct vb2_v4l2_buffer *src_buf, *dst_buf; - - src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); - dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); - ctx->coded_fmt_desc->ops->done(ctx, src_buf, dst_buf, result); - } - - v4l2_m2m_buf_done_and_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx, - result); + if (WARN_ON(!ctx->dev->building)) + return; + ctx->dev->compat_held = true; + ctx->dev->building->status = result == VB2_BUF_STATE_DONE ? 0 : -EIO; } void avd_job_finish(struct avd_ctx *ctx, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; - - pm_runtime_put_autosuspend(avd->dev); avd_job_finish_no_pm(ctx, result); } diff --git a/drivers/media/platform/apple/avd/avd-vp9.c b/drivers/media/platform/apple/avd/avd-vp9.c index 4cdcb6594c4d2..58b0106371f90 100644 --- a/drivers/media/platform/apple/avd/avd-vp9.c +++ b/drivers/media/platform/apple/avd/avd-vp9.c @@ -160,6 +160,7 @@ struct avd_vp9_frame_info { u32 interpolation_filter : 3; u32 flags; u64 timestamp; + dma_addr_t color; struct v4l2_vp9_segmentation seg; struct v4l2_vp9_loop_filter lf; }; @@ -171,10 +172,14 @@ struct avd_vp9_run { const struct v4l2_ctrl_vp9_compressed_hdr *prob_updates; }; -struct avd_vp9_ctx { +#define AVD_VP9_BANKS 2 + +/* Every DMA scratch mapping stays owned until PP retirement, including counts. */ +struct avd_vp9_bank { + bool busy; + bool updated; struct v4l2_vp9_frame_symbol_counts cnts; struct v4l2_vp9_frame_context probability_tables; - struct v4l2_vp9_frame_context frame_context[4]; struct avd_vp9_bufs { struct avd_buf inst; struct avd_buf az_left; @@ -194,6 +199,35 @@ struct avd_vp9_ctx { u8 submit_num; }; +struct avd_vp9_ctx { + struct v4l2_vp9_frame_context frame_context[4]; + struct avd_vp9_frame_info last; + unsigned int nbanks; + struct avd_vp9_bank bank[AVD_VP9_BANKS]; + struct avd_vp9_bank *b; +}; + +static bool avd_vp9_needs_counts(u32 flags) +{ + return (flags & V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX) && + !(flags & (V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE | + V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT)); +} + +static bool avd_vp9_ready(struct avd_ctx *ctx) +{ + struct avd_vp9_ctx *vp9_ctx = smp_load_acquire(&ctx->priv); + + if (!vp9_ctx) + return false; + /* Only a producer doing backward adaptation holds the next prepare. */ + for (unsigned int i = 0; i < vp9_ctx->nbanks; i++) + if (READ_ONCE(vp9_ctx->bank[i].busy) && + !READ_ONCE(vp9_ctx->bank[i].updated)) + return false; + return true; +} + static void set_refs(struct avd_ctx *ctx, struct avd_vp9_run *run) { const struct v4l2_ctrl_vp9_frame *frame = run->decode_params; @@ -233,6 +267,15 @@ static void set_refs(struct avd_ctx *ctx, struct avd_vp9_run *run) } } +/* Temporal MV input is the previous shown inter picture, not the new bank. */ +static bool avd_vp9_uses_prev_mv(u32 flags, const struct avd_vp9_frame_info *last) +{ + return !(flags & V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT) && + !(flags & V4L2_VP9_FRAME_FLAG_KEY_FRAME) && last->valid && + (last->flags & V4L2_VP9_FRAME_FLAG_SHOW_FRAME) && + !(last->flags & V4L2_VP9_FRAME_FLAG_KEY_FRAME); +} + static u32 make_flags1(struct avd_ctx *ctx, struct avd_vp9_run *run) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; @@ -284,12 +327,7 @@ static u32 make_flags1(struct avd_ctx *ctx, struct avd_vp9_run *run) flags |= VP9_SEG_ENABLED(frame->seg.flags & V4L2_VP9_SEGMENTATION_FLAG_ENABLED); - flags |= VP9_HAS_REF( - !(frame->flags & V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT) && - !(frame->flags & V4L2_VP9_FRAME_FLAG_KEY_FRAME) && - vp9_ctx->last.valid && - vp9_ctx->last.flags & V4L2_VP9_FRAME_FLAG_SHOW_FRAME && - !(vp9_ctx->last.flags & V4L2_VP9_FRAME_FLAG_KEY_FRAME)); + flags |= VP9_HAS_REF(avd_vp9_uses_prev_mv(frame->flags, &vp9_ctx->last)); flags |= VP9_SEG_RESET( frame->seg.flags & V4L2_VP9_SEGMENTATION_FLAG_ENABLED && @@ -307,7 +345,7 @@ static u32 seg_features(struct avd_ctx *ctx, struct avd_vp9_run *run, unsigned int segid) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; - const struct v4l2_vp9_segmentation *seg = &vp9_ctx->cur.seg; + const struct v4l2_vp9_segmentation *seg = &vp9_ctx->b->cur.seg; u32 feature_val = 0; int feature_id = 0; @@ -381,13 +419,15 @@ static void set_header(struct avd_ctx *ctx, struct avd_vp9_run *run) push(0, ""); push(0, ""); - pusha(vp9_ctx->bufs.counts.addr, "counts", 0); - pusha(vp9_ctx->bufs.probs.addr, "probs", 0); - pusha(vp9_ctx->bufs.above_info.addr, "above_info", 0); - pusha(vp9_ctx->bufs.seg.addr, "seg", 1); - pusha(vp9_ctx->bufs.seg.addr, "seg", 2); - pusha(vp9_ctx->bufs.color.addr, "color", 3); - pusha(vp9_ctx->bufs.color.addr, "color", 4); + pusha(vp9_ctx->b->bufs.counts.addr, "counts", 0); + pusha(vp9_ctx->b->bufs.probs.addr, "probs", 0); + pusha(vp9_ctx->b->bufs.above_info.addr, "above_info", 0); + pusha(vp9_ctx->b->bufs.seg.addr, "seg", 1); + pusha(vp9_ctx->b->bufs.seg.addr, "seg", 2); + pusha(vp9_ctx->b->bufs.color.addr, "color", 3); + pusha(avd_vp9_uses_prev_mv(frame->flags, &vp9_ctx->last) ? + vp9_ctx->last.color : vp9_ctx->b->bufs.color.addr, + "color", 4); push(VP9_Q_IDX(frame->quant.base_q_idx) | VP9_Q_DC_Y(frame->quant.delta_q_y_dc) | @@ -419,15 +459,15 @@ static void set_header(struct avd_ctx *ctx, struct avd_vp9_run *run) if (avd->variant->revision == 3) push(0, ""); if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(vp9_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(vp9_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(vp9_ctx->bufs.ip_above.addr, "ip_above", 0); - pusha(vp9_ctx->bufs.lf_above.addr, "lf_above", 0); + pusha(vp9_ctx->b->bufs.ip_above.addr, "ip_above", 0); + pusha(vp9_ctx->b->bufs.lf_above.addr, "lf_above", 0); /* no lf_above_info? */ pusha((u64)0, "hdr_e8_sps0_tile_addr_lsb8", 0); - pusha(vp9_ctx->bufs.lf_left.addr, "lf_left", 0); - pusha(vp9_ctx->bufs.lf_left_info.addr, "lf_left_info", 0); - pusha(vp9_ctx->bufs.az_left.addr, "az_left", 0); + pusha(vp9_ctx->b->bufs.lf_left.addr, "lf_left", 0); + pusha(vp9_ctx->b->bufs.lf_left_info.addr, "lf_left_info", 0); + pusha(vp9_ctx->b->bufs.az_left.addr, "az_left", 0); push(0, ""); @@ -507,7 +547,7 @@ static void set_tiles(struct avd_ctx *ctx, struct avd_vp9_run *run) offset += tile_size; size -= tile_size; - vp9_ctx->submit_num++; + vp9_ctx->b->submit_num++; } } @@ -523,18 +563,19 @@ static void update_ctx_cur_info(struct avd_vp9_ctx *vp9_ctx, struct avd_decoded_buffer *buf, const struct v4l2_ctrl_vp9_frame *dec_params) { - vp9_ctx->cur.valid = true; - vp9_ctx->cur.reference_mode = dec_params->reference_mode; - vp9_ctx->cur.interpolation_filter = dec_params->interpolation_filter; - vp9_ctx->cur.flags = dec_params->flags; - vp9_ctx->cur.timestamp = buf->base.vb.vb2_buf.timestamp; - vp9_ctx->cur.seg = dec_params->seg; - vp9_ctx->cur.lf = dec_params->lf; + vp9_ctx->b->cur.valid = true; + vp9_ctx->b->cur.reference_mode = dec_params->reference_mode; + vp9_ctx->b->cur.interpolation_filter = dec_params->interpolation_filter; + vp9_ctx->b->cur.flags = dec_params->flags; + vp9_ctx->b->cur.timestamp = buf->base.vb.vb2_buf.timestamp; + vp9_ctx->b->cur.color = vp9_ctx->b->bufs.color.addr; + vp9_ctx->b->cur.seg = dec_params->seg; + vp9_ctx->b->cur.lf = dec_params->lf; } static void update_ctx_last_info(struct avd_vp9_ctx *vp9_ctx) { - vp9_ctx->last = vp9_ctx->cur; + vp9_ctx->last = vp9_ctx->b->cur; } static void copy_vp9_frame_mv(struct avd_vp9_probs *avd_probs, @@ -563,14 +604,14 @@ static void init_probs(struct avd_ctx *ctx, const struct avd_vp9_run *run) { const struct v4l2_ctrl_vp9_frame *dec_params; struct avd_vp9_ctx *vp9_ctx = ctx->priv; - struct avd_vp9_probs *avd_probs = vp9_ctx->bufs.probs.cpu; + struct avd_vp9_probs *avd_probs = vp9_ctx->b->bufs.probs.cpu; const struct v4l2_vp9_segmentation *seg; const struct v4l2_vp9_frame_context *probs; bool intra_only; int count = 0; dec_params = run->decode_params; - probs = &vp9_ctx->probability_tables; + probs = &vp9_ctx->b->probability_tables; seg = &dec_params->seg; memset(avd_probs, 0, sizeof(*avd_probs)); @@ -651,10 +692,9 @@ static int validate_dec_params(struct avd_ctx *ctx, return 0; } -static int avd_vp9_alloc_bufs(struct avd_ctx *ctx) +static int avd_vp9_alloc_bufs(struct avd_ctx *ctx, struct avd_vp9_bank *bank) { struct avd_dev *avd = ctx->dev; - struct avd_vp9_ctx *vp9_ctx = ctx->priv; int ret, w, h, bit_depth; w = fmt_width(ctx); h = fmt_height(ctx); @@ -663,62 +703,62 @@ static int avd_vp9_alloc_bufs(struct avd_ctx *ctx) 10 : 8; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(avd, &bank->bufs.inst, fifo_size()); if (ret) return ret; if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) { - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(avd, &bank->bufs.pipe_state, 0x200); if (ret) return ret; } - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.probs, + ret = avd_buf_alloc(avd, &bank->bufs.probs, sizeof(struct avd_vp9_probs)); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.counts, + ret = avd_buf_alloc(avd, &bank->bufs.counts, sizeof(struct avd_vp9_frame_symbol_counts)); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.color, + ret = avd_buf_alloc(avd, &bank->bufs.color, DIV_ROUND_UP(w, 16) * DIV_ROUND_UP(h, 64) * 144); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.ip_above, + ret = avd_buf_alloc(avd, &bank->bufs.ip_above, DIV_ROUND_UP(w, 16) * 4 * bit_depth + (VP9_MAX_TILE_COLS - 1) * 128); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.lf_above, + ret = avd_buf_alloc(avd, &bank->bufs.lf_above, DIV_ROUND_UP(w, 8) * 16 * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.seg, DIV_ROUND_UP(w, 8) * 24); + ret = avd_buf_alloc(avd, &bank->bufs.seg, DIV_ROUND_UP(w, 8) * 24); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.lf_left, + ret = avd_buf_alloc(avd, &bank->bufs.lf_left, DIV_ROUND_UP(h, 8) * 16 * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.lf_left_info, + ret = avd_buf_alloc(avd, &bank->bufs.lf_left_info, DIV_ROUND_UP(h, 64) * 16); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.az_left, + ret = avd_buf_alloc(avd, &bank->bufs.az_left, bit_depth * 36 * DIV_ROUND_UP(h, 8) + bit_depth * 18 * DIV_ROUND_UP(h, 16)); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.above_info, + ret = avd_buf_alloc(avd, &bank->bufs.above_info, DIV_ROUND_UP(w, 64) * 288 + (VP9_MAX_TILE_COLS - 1) * 128); if (ret) @@ -754,18 +794,40 @@ static int avd_vp9_run_preamble(struct avd_ctx *ctx, struct avd_vp9_run *run) run->decode_params = dec_params; - vp9_ctx->cur.tx_mode = run->prob_updates->tx_mode; + if (avd_backend_is_qfw(ctx->dev)) { + struct avd_vp9_bank *bank = NULL; + + for (unsigned int i = 0; i < vp9_ctx->nbanks; i++) + if (!vp9_ctx->bank[i].busy) { + bank = &vp9_ctx->bank[i]; + break; + } + if (!bank) + return -EBUSY; + vp9_ctx->b = bank; + WRITE_ONCE(bank->busy, true); + WRITE_ONCE(bank->updated, false); + bank->last = vp9_ctx->last; + vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf)->bank = + bank - vp9_ctx->bank + 1; + } + + vp9_ctx->b->cur.tx_mode = run->prob_updates->tx_mode; fctx_idx = v4l2_vp9_reset_frame_ctx(dec_params, vp9_ctx->frame_context); - vp9_ctx->cur.frame_context_idx = fctx_idx; + vp9_ctx->b->cur.frame_context_idx = fctx_idx; - vp9_ctx->probability_tables = vp9_ctx->frame_context[fctx_idx]; - v4l2_vp9_fw_update_probs(&vp9_ctx->probability_tables, + vp9_ctx->b->probability_tables = vp9_ctx->frame_context[fctx_idx]; + v4l2_vp9_fw_update_probs(&vp9_ctx->b->probability_tables, run->prob_updates, dec_params); return 0; } +static void avd_vp9_update_probs(struct avd_ctx *ctx, + struct avd_vp9_bank *bank, + enum vb2_buffer_state result); + static int avd_vp9_run(struct avd_ctx *ctx) { struct avd_vp9_run run; @@ -797,10 +859,17 @@ static int avd_vp9_run(struct avd_ctx *ctx) update_ctx_cur_info(vp9_ctx, dst, run.decode_params); set_header(ctx, &run); - vp9_ctx->submit_num = 0; + vp9_ctx->b->submit_num = 0; set_tiles(ctx, &run); avd_run_postamble(ctx, &run.base); + if (avd_backend_is_qfw(ctx->dev)) { + /* Forward updates need no hardware result; publish before next prepare. */ + if (!avd_vp9_needs_counts(vp9_ctx->b->cur.flags)) + avd_vp9_update_probs(ctx, vp9_ctx->b, VB2_BUF_STATE_DONE); + update_ctx_last_info(vp9_ctx); + ctx->job_fifo = vp9_ctx->b->bufs.inst.addr; + } return avd_submit_job(ctx); } @@ -812,16 +881,19 @@ static int avd_vp9_run(struct avd_ctx *ctx) memcpy((p1)->skip, (p2)->skip, sizeof((p1)->skip)); \ } while (0) -static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, - struct vb2_v4l2_buffer *dst_buf, - enum vb2_buffer_state result) +static void avd_vp9_update_probs(struct avd_ctx *ctx, + struct avd_vp9_bank *bank, + enum vb2_buffer_state result) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; unsigned int fctx_idx; + if (bank->updated) + return; + /* v4l2-specific stuff */ if (result == VB2_BUF_STATE_ERROR) - goto out_update_last; + goto out; /* * vp9 stuff @@ -832,19 +904,20 @@ static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, * but it makes no sense to perform all the activities from the first "if" * there if we actually are not refreshing the frame context. On top of that, * because of 6.2 uncompressed_header() whenever error_resilient_mode == 1, - * refresh_frame_context == 0. Consequently, if we don't jump to out_update_last + * refresh_frame_context == 0. Consequently, if we don't jump to out * it means error_resilient_mode must be 0. */ - if (!(vp9_ctx->cur.flags & V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX)) - goto out_update_last; + if (!(bank->cur.flags & V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX) || + (bank->cur.flags & V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT)) + goto out; - fctx_idx = vp9_ctx->cur.frame_context_idx; + fctx_idx = bank->cur.frame_context_idx; - if (!(vp9_ctx->cur.flags & V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE)) { + if (avd_vp9_needs_counts(bank->cur.flags)) { /* error_resilient_mode == 0 && frame_parallel_decoding_mode == 0 */ struct v4l2_vp9_frame_context *probs = - &vp9_ctx->probability_tables; - bool frame_is_intra = vp9_ctx->cur.flags & + &bank->probability_tables; + bool frame_is_intra = bank->cur.flags & (V4L2_VP9_FRAME_FLAG_KEY_FRAME | V4L2_VP9_FRAME_FLAG_INTRA_ONLY); struct tx_and_skip { @@ -866,12 +939,12 @@ static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, if (frame_is_intra) copy_tx_and_skip(probs, tx_skip); - counts = &vp9_ctx->cnts; + counts = &bank->cnts; v4l2_vp9_adapt_coef_probs( probs, counts, - !vp9_ctx->last.valid || - vp9_ctx->last.flags & + !bank->last.valid || + bank->last.flags & V4L2_VP9_FRAME_FLAG_KEY_FRAME, frame_is_intra); if (!frame_is_intra) { @@ -887,7 +960,7 @@ static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, u32 class0_hp[2][2]; u32 hp[2][2]; - cnts = vp9_ctx->bufs.counts.cpu; + cnts = bank->bufs.counts.cpu; for (i = 0; i < ARRAY_SIZE(cnts->tx16p); ++i) memcpy(tx16p[i], cnts->tx16p[i], @@ -922,47 +995,81 @@ static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, /* load_probs2() already done */ v4l2_vp9_adapt_noncoef_probs( - &vp9_ctx->probability_tables, counts, - vp9_ctx->cur.reference_mode, - vp9_ctx->cur.interpolation_filter, - vp9_ctx->cur.tx_mode, vp9_ctx->cur.flags); + &bank->probability_tables, counts, + bank->cur.reference_mode, + bank->cur.interpolation_filter, + bank->cur.tx_mode, bank->cur.flags); } } /* 6.1.2 refresh_probs(): save_probs(fctx_idx) */ - vp9_ctx->frame_context[fctx_idx] = vp9_ctx->probability_tables; + vp9_ctx->frame_context[fctx_idx] = bank->probability_tables; -out_update_last: - update_ctx_last_info(vp9_ctx); +out: + WRITE_ONCE(bank->updated, true); } -static noinline_for_stack void -avd_init_v4l2_vp9_count_tbl(struct avd_ctx *ctx) + +static void avd_vp9_vp_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst) +{ + struct avd_vp9_ctx *vp9_ctx = ctx->priv; + unsigned int id = vb2_to_avd_decoded_buf(&dst->vb2_buf)->bank; + + if (id && id <= vp9_ctx->nbanks) + avd_vp9_update_probs(ctx, &vp9_ctx->bank[id - 1], VB2_BUF_STATE_DONE); +} + +static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src, + struct vb2_v4l2_buffer *dst, enum vb2_buffer_state result) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; - struct avd_vp9_frame_symbol_counts *cnts = vp9_ctx->bufs.counts.cpu; + struct avd_decoded_buffer *buf = vb2_to_avd_decoded_buf(&dst->vb2_buf); + struct avd_vp9_bank *bank; + + if (!buf->bank) { + if (avd_backend_is_qfw(ctx->dev)) + return; + vp9_ctx->b->last = vp9_ctx->last; + WRITE_ONCE(vp9_ctx->b->updated, false); + avd_vp9_update_probs(ctx, vp9_ctx->b, result); + update_ctx_last_info(vp9_ctx); + return; + } + if (buf->bank > vp9_ctx->nbanks) + return; + bank = &vp9_ctx->bank[buf->bank - 1]; + buf->bank = 0; + /* Also covers old queue firmware and preparation/submission failures. */ + avd_vp9_update_probs(ctx, bank, result); + WRITE_ONCE(bank->busy, false); +} + +static noinline_for_stack void +avd_init_v4l2_vp9_count_tbl(struct avd_vp9_bank *bank) +{ + struct avd_vp9_frame_symbol_counts *cnts = bank->bufs.counts.cpu; int coeff_cnts = 0, eob_cnts = 0; - vp9_ctx->cnts.intra_inter = &cnts->intra_inter; - vp9_ctx->cnts.y_mode = &cnts->y_mode; - vp9_ctx->cnts.uv_mode = &cnts->uv_mode; - vp9_ctx->cnts.comp = &cnts->comp; - vp9_ctx->cnts.comp_ref = &cnts->comp_ref; - vp9_ctx->cnts.single_ref = &cnts->single_ref; - vp9_ctx->cnts.filter = &cnts->filter; - vp9_ctx->cnts.mv_mode = &cnts->mv_mode; - vp9_ctx->cnts.mv_joint = &cnts->mv_joint; + bank->cnts.intra_inter = &cnts->intra_inter; + bank->cnts.y_mode = &cnts->y_mode; + bank->cnts.uv_mode = &cnts->uv_mode; + bank->cnts.comp = &cnts->comp; + bank->cnts.comp_ref = &cnts->comp_ref; + bank->cnts.single_ref = &cnts->single_ref; + bank->cnts.filter = &cnts->filter; + bank->cnts.mv_mode = &cnts->mv_mode; + bank->cnts.mv_joint = &cnts->mv_joint; /* all of the mv use a different structure, so they must all be copied */ - vp9_ctx->cnts.tx8p = &cnts->tx8p; + bank->cnts.tx8p = &cnts->tx8p; /* avd also uses "u32 tx16p[2][3]" instead of "u32 tx16p[2][4]" */ - vp9_ctx->cnts.tx32p = &cnts->tx32p; + bank->cnts.tx32p = &cnts->tx32p; /* it would suck if we need to reverse partition */ - vp9_ctx->cnts.partition = &cnts->partition; + bank->cnts.partition = &cnts->partition; /* these should work */ - vp9_ctx->cnts.skip = &cnts->skip; + bank->cnts.skip = &cnts->skip; for (int t = 0; t < 4; t++) for (int i = 0; i < 2; i++) @@ -970,36 +1077,46 @@ avd_init_v4l2_vp9_count_tbl(struct avd_ctx *ctx) for (int k = 0; k < 6; k++) { int max_l = (k == 0) ? 3 : 6; for (int l = 0; l < max_l; l++) { - vp9_ctx->cnts.coeff[t][i][j][k][l] = + bank->cnts.coeff[t][i][j][k][l] = (u32 (*)[3])&cnts->ref_cnt[coeff_cnts]; coeff_cnts += 3; - vp9_ctx->cnts.eob[t][i][j][k][l][0] = + bank->cnts.eob[t][i][j][k][l][0] = &cnts->eob_0[eob_cnts++]; - vp9_ctx->cnts.eob[t][i][j][k][l][1] = + bank->cnts.eob[t][i][j][k][l][1] = &cnts->ref_cnt[coeff_cnts++]; } } } -static void avd_vp9_stop(struct avd_ctx *ctx) +static void avd_vp9_free(struct avd_ctx *ctx, struct avd_vp9_ctx *vp9_ctx) { - struct avd_vp9_ctx *vp9_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; - avd_buf_free(avd, &vp9_ctx->bufs.pipe_state); - avd_buf_free(avd, &vp9_ctx->bufs.inst); - avd_buf_free(avd, &vp9_ctx->bufs.probs); - avd_buf_free(avd, &vp9_ctx->bufs.counts); - avd_buf_free(avd, &vp9_ctx->bufs.seg); - avd_buf_free(avd, &vp9_ctx->bufs.color); - avd_buf_free(avd, &vp9_ctx->bufs.ip_above); - avd_buf_free(avd, &vp9_ctx->bufs.lf_above); - avd_buf_free(avd, &vp9_ctx->bufs.above_info); - avd_buf_free(avd, &vp9_ctx->bufs.az_left); - avd_buf_free(avd, &vp9_ctx->bufs.lf_left_info); - avd_buf_free(avd, &vp9_ctx->bufs.lf_left); + ctx->max_frames = 1; + for (unsigned int i = 0; i < vp9_ctx->nbanks; i++) { + struct avd_vp9_bank *bank = &vp9_ctx->bank[i]; + + avd_buf_free(avd, &bank->bufs.pipe_state); + avd_buf_free(avd, &bank->bufs.inst); + avd_buf_free(avd, &bank->bufs.probs); + avd_buf_free(avd, &bank->bufs.counts); + avd_buf_free(avd, &bank->bufs.seg); + avd_buf_free(avd, &bank->bufs.color); + avd_buf_free(avd, &bank->bufs.ip_above); + avd_buf_free(avd, &bank->bufs.lf_above); + avd_buf_free(avd, &bank->bufs.above_info); + avd_buf_free(avd, &bank->bufs.az_left); + avd_buf_free(avd, &bank->bufs.lf_left_info); + avd_buf_free(avd, &bank->bufs.lf_left); + } kfree(vp9_ctx); +} + +static void avd_vp9_stop(struct avd_ctx *ctx) +{ + /* drain() excludes readiness callbacks before private storage is freed. */ + avd_vp9_free(ctx, ctx->priv); ctx->priv = NULL; } @@ -1012,17 +1129,22 @@ static int avd_vp9_start(struct avd_ctx *ctx) if (!vp9_ctx) return -ENOMEM; - ctx->priv = vp9_ctx; - ret = avd_vp9_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; - - avd_init_v4l2_vp9_count_tbl(ctx); + vp9_ctx->nbanks = avd_qfw_has_vp_done(ctx->dev) ? AVD_VP9_BANKS : 1; + vp9_ctx->b = &vp9_ctx->bank[0]; + for (unsigned int i = 0; i < vp9_ctx->nbanks; i++) { + ret = avd_vp9_alloc_bufs(ctx, &vp9_ctx->bank[i]); + if (ret) + goto err_free_ctx; + avd_init_v4l2_vp9_count_tbl(&vp9_ctx->bank[i]); + } + ctx->max_frames = vp9_ctx->nbanks; + /* job_ready can run while STREAMON is still initializing this codec. */ + smp_store_release(&ctx->priv, vp9_ctx); return 0; err_free_ctx: - avd_vp9_stop(ctx); + avd_vp9_free(ctx, vp9_ctx); return ret; } @@ -1049,6 +1171,13 @@ static enum avd_image_fmt avd_vp9_get_image_fmt(struct avd_ctx *ctx, #undef BIT_DEPTH } +static unsigned int avd_vp9_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_vp9_ctx *priv = ctx->priv; + + return max_t(int, priv->b->submit_num, 1); +} + static void avd_vp9_submit(struct avd_ctx *ctx) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; @@ -1065,7 +1194,7 @@ static void avd_vp9_submit(struct avd_ctx *ctx) AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx) | AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots), avd->ctrl + avd->variant->submit_offset); - for (int i = 0; i < vp9_ctx->submit_num - 1; i++) + for (int i = 0; i < vp9_ctx->b->submit_num - 1; i++) writel(AVD_OP_EXEC | submit_mask | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx) | AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots), @@ -1077,6 +1206,9 @@ const struct avd_coded_fmt_ops avd_vp9_fmt_ops = { .stop = avd_vp9_stop, .run = avd_vp9_run, .done = avd_vp9_done, + .vp_done = avd_vp9_vp_done, + .ready = avd_vp9_ready, .submit = avd_vp9_submit, + .pp_kicks = avd_vp9_pp_kicks, .get_image_fmt = avd_vp9_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 153a462320c4a..2da75d833b75a 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -19,6 +19,11 @@ #include #include +#include "avd-pm.h" +#include + +#include "avd-queue.h" + #include #include #include @@ -123,6 +128,8 @@ struct avd_decoded_buffer { /* Pending CPU fill, retained with the buffer until dequeue. */ size_t grey_chroma_offset; bool grey_chroma_10bit; + /* Codec scratch bank + 1 of the picture decoded into this buffer */ + u8 bank; union { struct avd_vp9_decoded_buffer_info vp9; @@ -144,12 +151,18 @@ struct avd_coded_fmt_ops { void (*adjust_decoded_fmt)(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp); void (*submit)(struct avd_ctx *ctx); + /* PP EXEC commands the last built frame needs (submit's count), >= 1. */ + unsigned int (*pp_kicks)(struct avd_ctx *ctx); int (*start)(struct avd_ctx *ctx); void (*stop)(struct avd_ctx *ctx); int (*run)(struct avd_ctx *ctx); void (*done)(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, struct vb2_v4l2_buffer *dst_buf, enum vb2_buffer_state result); + /* Called under queue_lock: CPU preparation may read prior VP results. */ + bool (*ready)(struct avd_ctx *ctx); + /* queue_mutex held, after a tagged VP DONE; DMA banks remain owned. */ + void (*vp_done)(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst); int (*try_ctrl)(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl); enum avd_image_fmt (*get_image_fmt)(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl); @@ -173,20 +186,29 @@ struct avd_coded_fmt_desc { struct v4l2_frmsize_stepwise frmsize; const struct avd_ctrls *ctrls; const struct avd_coded_fmt_ops *ops; + const struct avd_codec_frame_ops *frame_ops; unsigned int num_decoded_fmts; const struct avd_decoded_fmt_desc *decoded_fmts; u32 subsystem_flags; unsigned int capability; }; +struct avd_apple_hw; + struct avd_variant { + const struct avd_apple_hw *apple; + size_t fp_size; unsigned int vp_slots[4]; unsigned int vp_slot_base[4]; unsigned int fifo_slots; unsigned int capabilities; + /* Codecs advertised only when the firmware CODECS trailer lists them. */ + unsigned int fw_caps_required; void (*configure_stream)(struct avd_dev *avd, dma_addr_t addr, u8 fifo_idx, u32 vp_slot); const char* fw_name; + /* Older image tried when fw_name is not installed (legacy transport). */ + const char *fw_name_fallback; unsigned char revision; /* the same as the device tree */ /* just for convenience */ u32 vp_slot_offset; @@ -196,23 +218,16 @@ struct avd_variant { unsigned int quirks; }; -/* Lifecycle of the one m2m job the device runs at a time. */ -enum avd_job_state { - AVD_JOB_IDLE, - AVD_JOB_RUNNING, /* inside avd_device_run() */ - AVD_JOB_SUBMITTED, /* handed to the hardware, watchdog armed */ - AVD_JOB_FINISHING, /* claimed by the IRQ handler or the watchdog */ -}; - struct avd_dev { struct device *dev; + struct avd_pm pm; struct v4l2_device v4l2_dev; struct media_device mdev; struct video_device vdev; struct v4l2_m2m_dev *m2m_dev; struct platform_device *pdev; - const struct firmware *fw; /* fw is lost on suspend */ + const struct firmware *fw; /* separately installed open firmware */ void __iomem *code; void __iomem *mbox; @@ -225,32 +240,51 @@ struct avd_dev { struct mutex vdev_lock; - /* The current job; see avd_device_run(). Protected by job_lock. */ - spinlock_t job_lock; - enum avd_job_state job_state; - struct avd_ctx *job_ctx; - u64 job_seq; - bool job_end_sent; - /* The current job's VP finished and its PP was kicked. */ - bool job_vp_done; - bool job_pending; - enum vb2_buffer_state job_pending_result; - - struct reset_control *rstc; - int irq; - /* Recovery budget; a retired decoder fails every job until reload. */ - unsigned long recover_window_start; - unsigned int recoveries; + /* queue_lock protects slot state/tag and job_ready observations. */ + spinlock_t queue_lock; + /* Serializes CPU preparation, submission, retirement and reset. */ + struct mutex queue_mutex; + struct delayed_work queue_work; + struct list_head contexts; + /* One m2m dispatch reservation; never identifies firmware-owned work. */ + /* + * m2m runs one device_run at a time until job_finish; a job that finds + * no free slot waits here and is restarted when a frame retires. + */ + struct avd_ctx *deferred_ctx; + struct avd_frame frames[AVD_MAX_FRAMES]; + struct avd_buf *frame_fp[AVD_MAX_FRAMES]; + u64 frame_seq; + unsigned int inflight; + bool stopping; + /* Retired after a failed recovery, until module reload. */ + bool faulted; + /* Recovery pacing: when the last one ran, and a sliding storm count. */ unsigned long last_recover; - bool retired; - /* DMA blocked at the DART (retired or resting); detached at remove. */ - bool dma_parked; - /* Out of recovery budget: frames fail without hardware access until then. */ + unsigned long storm_start; + unsigned int storm_count; + /* Out of recovery budget: keep DMA parked until the window ends. */ unsigned long rest_until; - bool irq_parked; + u64 fw_generation; + const struct avd_backend_ops *backend; + void *backend_priv; + /* Only the open-firmware compatibility path uses these. */ + struct avd_frame *building; + /* Open firmware queue ABI 2 state, NULL when the node has no SRAM. */ + struct avd_qfw *qfw; + struct avd_frame *compat_active; + /* The active legacy frame's VP finished and its PP was kicked. */ + bool compat_pp_kicked; + /* Context of the last legacy submission, for VP DONE without a frame. */ + struct avd_ctx *compat_ctx; + bool compat_held; + + struct reset_control *rstc; const struct avd_variant *variant; unsigned int capabilities; + /* What the loaded image's CODECS trailer (or its absence) allows. */ + unsigned int fw_capabilities; }; struct avd_segment { @@ -268,13 +302,12 @@ struct avd_buf { void *cpu; dma_addr_t addr; size_t size; + struct avd_dma_release *release; }; struct avd_ctx { struct v4l2_fh fh; struct avd_dev *dev; - /* A frame failed in hardware: fail every job until streaming restarts. */ - bool broken; struct v4l2_format coded_fmt; struct v4l2_format decoded_fmt; @@ -284,14 +317,26 @@ struct avd_ctx { enum avd_image_fmt image_fmt; bool decomp; - struct delayed_work watchdog_work; - /* job_seq of the job watchdog_work was armed for */ - u64 wd_seq; + struct list_head queue_node; + struct work_struct submit_work; + bool draining; + /* Consecutive frames that failed in hardware; reset by a good frame. */ + unsigned int hw_errors; + /* Too many hardware errors: fail every frame until streaming restarts. */ + bool broken; + bool session_started; + u64 session_generation; + enum avd_frame_dep next_dep; + u32 client_id; void *priv; struct avd_comp comp; int fifo_idx; + /* Pictures of this context the queue may hold at once (0 or 1: one). */ + unsigned int max_frames; + /* VP to PP FIFO for the job being built; 0 means ctx->inst. */ + dma_addr_t job_fifo; struct avd_job job; struct avd_buf inst; }; @@ -347,6 +392,28 @@ static inline struct avd_ctx *file_to_ctx(struct file *filp) /* hw stuff */ int avd_boot(struct avd_dev *avd); +int avd_queue_init(struct avd_dev *avd); +void avd_queue_cleanup(struct avd_dev *avd); +void avd_queue_run(void *priv); +int avd_queue_ready(void *priv); +void avd_queue_abort(void *priv); +void avd_queue_drain(struct avd_ctx *ctx); +void avd_queue_add_ctx(struct avd_ctx *ctx); +void avd_queue_del_ctx(struct avd_ctx *ctx); +int avd_open_build_program(struct avd_ctx *ctx); +extern const struct avd_backend_ops avd_open_backend; +extern const struct avd_backend_ops avd_apple_backend; +extern const struct avd_backend_ops avd_qfw_backend; +struct avd_qfw; +int avd_qfw_probe(struct avd_dev *avd); +void avd_qfw_select(struct avd_dev *avd); +void avd_qfw_prepare_boot(struct avd_dev *avd); +void avd_qfw_debug(struct avd_dev *avd, struct avd_frame *f, const char *where); +bool avd_backend_is_qfw(const struct avd_dev *avd); +bool avd_qfw_has_vp_done(const struct avd_dev *avd); +int avd_backend_select(struct avd_dev *avd); +bool avd_mbox_pop(struct avd_dev *avd, u32 *msg); +int avd_backend_boot(struct avd_dev *avd); void avd_shutdown(struct avd_dev *avd); void avd_status(struct avd_dev *avd, u32 vp); diff --git a/drivers/media/platform/apple/avd/tests/.kunitconfig b/drivers/media/platform/apple/avd/tests/.kunitconfig new file mode 100644 index 0000000000000..c079bc44af531 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/.kunitconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +CONFIG_KUNIT=y +CONFIG_COMPILE_TEST=y +CONFIG_MEDIA_SUPPORT=y +# CONFIG_MEDIA_SUPPORT_FILTER is not set +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_OF=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_WERROR=y diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst new file mode 100644 index 0000000000000..d2b10320f2b75 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -0,0 +1,233 @@ +Queued open-firmware regressions +=============================== + +Run compilation in a controller buildbox slot. These tests use the current +production C function bodies; allocation, buffer addresses, V4L2 scheduling and +MMIO are explicit host fixtures. Linux UAPI headers and an ASan/UBSan-capable C +compiler are required:: + + sh tests/run-host.sh /tmp/avd-queue-host + python3 tests/program-test.py /tmp/avd-program + python3 tests/dispatch-test.py /tmp/avd-dispatch + python3 tests/recovery-test.py /tmp/avd-recovery + +The program test runs the H.264 header/slice emitters, open-program builder, +t8112_configure_stream(), queued descriptor submit and SRAM streaming code. +It generates a 1920x1088 IDR program with two I slices and fixture controls/IOVAs, plus +ABI 2 descriptors for FIFO indices 0 through 14. It verifies busy-mailbox +backpressure, the doorbell address, LSR FIFO addressing, decoder IRQ masks, +ring wrap and free-running u32 counter wrap. It checks mailbox draining, +full-tag rejection and fatal completion ownership until quiesced recovery. It also checks that legacy VP +EXEC words retain their existing values. Input SHA256 manifests and generated +C sources accompany the binaries. + +For T8112 (host revision 4, firmware v3), VP EXEC start is 0x2b000200 plus +FIFO_IDX, and END is 0x2b000400 plus FIFO_IDX. VP programs retain the legacy +zero low nibble. PP start is 0x2b00020f plus FIFO_IDX: its low nibble carries +the hardware FIFO mask, 15. Do not replace the VP's low nibble with the PP mask +or confuse host revision IDs with firmware image versions. + +The dispatch test compiles the production submit/retire functions and V4L2 +core detach/held-buffer helpers. It checks early handoff without completing +DMA-owned buffers, held slices, two contexts, deferred capacity, stale work, +prepare/PM/abort failures and exactly-once scheduler release. The matching +V4L2 m2m core source must be present in the kernel checkout. + +Routed CM3 test +-------------- + +Use avd-fw PR #4's linear ABI 2 implementation (baseline commit +7f2f41369d14e0deccbac9b7d2b7c9070e48e66d). Build its images and install the +optional Unicorn Python dependency. The firmware checkout supplies the existing +transport-test.py harness:: + + python3 tests/unicorn-queue-test.py /path/to/avd-fw \ + /path/to/avd-fw/build/avd-fw-v3-t1.bin /tmp/avd-program --broken + +The baseline must reproduce both missing CM3 decoder interrupt routing and, +when those routes are repaired in the fixture, the incorrect mailbox status +EMPTY test. A firmware fix must program the queue's CM3 IRQ router and use +FIFO status bit 17, rather than IRQ-source bit 2, for outbox EMPTY. Both fixes +are in ``aurora-silicon/avd-fw`` from commit ``50d7f869``. With that firmware +built:: + + python3 tests/unicorn-queue-test.py /path/to/fixed-avd-fw \ + /path/to/fixed-avd-fw/build/avd-fw-v3-t1.bin /tmp/avd-program + +The test boots the actual reset vector, checks NVIC and CM3 router masks before +calling IRQ wrappers, consumes the production Linux descriptor/program, and +raises VP/PP DONE only after the modeled program/kick. It requires descriptor +state COMPLETE and exactly the expected tagged mailbox-1 notification at FIFO +0, 1 and 14. The firmware's original all-image transport tests remain useful +for refill, ordering, backpressure and error behavior. + +Unicorn models decoder events and memory; it does not decode pixels or prove +silicon register behavior, DMA safety, throughput or decoded output correctness. +A controller-approved, single-request hardware decode and software framemd5 +comparison remain necessary. The accompanying firmware patch changes queue +mode only; apply it to the pinned open firmware source and retain its MIT +license. The unmodified firmware image cannot produce tagged completions with +the corrected mailbox-status model. + +The FIFO status and IRQ-router geometry follow the public MIT emulator at +https://github.com/eiln/avd/blob/main/avd_emu.py. The pinned avd-fw source's +Tansy routing independently identifies the same banks and bit positions. + +Concurrent pictures and DMA addresses +------------------------------------ + +``program-test.py`` also generates per-image subdirectories for v3-t0, v3-t1, +v5-t0 and v5-t2. It runs the production H.264 bank allocator and bank selection, +uses both banks of eight fixture contexts, and emits IDR, P and B pictures. +Every scratch, output, compressed-reference and MV address passed to an address +emitter must be nonzero. The emitted address locations/encodings are exported +alongside the actual descriptor and configure_stream MMIO snapshot for each +slot. Coded-data operands are checked separately by the routed test. Seven +fault-injection runs zero each bank-1 buffer address in turn and must fail. +A missing saved FIFO is rejected at every valid slot before the doorbell, +including when the context contains a different nonzero fallback address. + +Build the firmware with the busy-VP scheduling fix, then run:: + + for image in v3-t0 v3-t1 v5-t0 v5-t2; do + python3 tests/unicorn-inflight-test.py /path/to/avd-fw \ + /path/to/avd-fw/build/avd-fw-$image.bin /tmp/avd-program + done + +This test boots each image's reset vector, checks router and NVIC masks, replays +production host MMIO, and verifies every emitted DMA operand at its recorded +program location. It holds VP DONE, PP DONE and the completion mailbox to test +ownership and backpressure. Workloads range from 2 to 15 pictures. On the +15-FIFO images all requested pictures are posted before any VP DONE. v5-t0 +advertises only seven physical FIFOs: workloads above seven use a seven-slot +window with slot reuse and a new full completion tag. This is not evidence of +8..15 simultaneous hardware FIFOs on that image. H.264 retains two banks per +context; larger windows represent multiple contexts. + +The decoder model asserts that a VP cannot receive the next picture's START +before its current picture's DONE. With the old firmware this reproduces at +slot 1; ``--before`` requires that failure instead of treating it as a pass. +The fixed scheduler waits at that boundary while retaining VP/PP overlap and +strict global SRAM program order. The native firmware test also covers +independent VPs and ordered PP completion. + +The live-VP assertion is a conservative decoder ownership model. It proves the +software feed ordering and its correction, not the undocumented silicon cause +of a DART fault. A hardware retest remains necessary to connect this scheduling +defect to a reported zero-address DMA and to verify pixels and performance. + +VP9 probability dependencies and picture banks +--------------------------------------------- + +``vp9-test.py`` compiles the production VP9 codec and V4L2 probability helpers. +It exercises all combinations of the three dependency flags, real backward +adaptation with synthetic counts, independent forward updates, per-bank count +pointers, bank exhaustion, PP retirement while another bank is current, and +old-firmware fallback. It also emits two VP9 programs and their production queue +descriptors/configure_stream snapshots for ``unicorn-vp9-test.py``:: + + python3 tests/vp9-test.py /tmp/avd-vp9 + python3 tests/unicorn-vp9-test.py /path/to/avd-fw \ + /path/to/avd-fw/build/avd-fw-v3-t1.bin /tmp/avd-vp9 + +For the independent legacy comparison, run in a kernel Git checkout containing +the public pre-change revision:: + + AVD_VP9_BASELINE_REF= python3 tests/vp9-test.py /tmp/avd-vp9 + +This builds separate before/after executables and compares the complete legacy +programs and saved probability contexts, including key, inter, frame-parallel, +non-refreshing and error-resilient frames. No firmware capture is used. + +The flags below describe whether a picture *produces* a backward-adaptation +dependency for the next prepare. A non-refreshing successor still consumes the +saved context of an earlier refreshing picture. No CPU can read that context +before that producer's counts have been adapted. + +================= ============= =============== ============================ +refresh_frame_ctx parallel_mode error_resilient Release next prepare +================= ============= =============== ============================ +0 any any During prepare +1 1 any During prepare (forward only) +1 0 1 During prepare (no refresh) +1 0 0 After tagged VP DONE/adaptation +================= ============= =============== ============================ + +Preparation, VP callbacks and PP retirement run under queue_mutex. Readiness +checks only observe the published dependency state under queue_lock. Counts +come from coherent allocations and the VP callback uses dma_rmb() before reading +them. Each bank has its own count-pointer table, forward probabilities, header +snapshot, previous-frame flags, FIFO and all DMA scratch. Only PP retirement +releases a bank, and it selects the bank saved with CAPTURE rather than the +context's current builder. A duplicate VP event cannot adapt twice. Failure and +reset retirement never adapt partial counts. + +The temporal-MV input color address follows the preceding picture across banks; +the current color output belongs to the new bank. The public MIT VP9 builder at +https://github.com/eiln/avd/blob/main/avid/vp9/halv3.py alternates these operands +(``pps2_tile_addrs``); the single-bank legacy driver used one address for both. +A stream using old queue firmware retains one bank and adapts after full +completion. The ordinary legacy transport and its instruction builder are +unchanged. + +The Unicorn test boots the real reset vector and routes VP8/PP IRQs. It checks +that VP(1) is issued while PP(0) owns its FIFO, and that PP(1) waits for PP(0). +It repeats with a blocked VP notification and holds the outbox across PP DONE. +The original H.264 routed tests exercise descriptors without the optional flag +and must retain their single completion per picture. + +Optional queue ABI 2 VP notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The feature is in ``aurora-silicon/avd-fw`` from commit +``bf49a31b1036c72be5fdc37eb8232f6c953e0626`` (queue feature bit 2). +This is a backwards-compatible optional ABI 2 feature: + +* Info feature bit 2 advertises tagged VP DONE. Without it the host keeps VP9 + at one picture per stream. Features 0/1 and the ordinary completion stay as + before. +* Host descriptor flag bit 1 requests one VP event. An old host leaves it zero + and receives no additional tokens. Reserved words must still start at zero. +* After VP DONE, firmware writes the full submitted tag at descriptor +0x38, + DMB, writes 1 at +0x3c, DMB, then sends + ``0x54540000 | (tag & 0xfff0) | index``. These records live independently of + the PP completion_tag. The host validates the prefix, marker, short sequence, + full saved tag and submitted ownership before recording the event. +* Outbox backpressure retains the VP event. A successful opted-in picture's PP + completion cannot be notified before its VP event. Errors may finish without + a VP event and require the existing reset/ownership handling. +* Neither the VP token nor its CPU callback permits buffer, descriptor, FIFO or + bank reuse. Only the original tagged PP completion does that. A missing VP + event remains bounded by the original per-picture watchdog. + +AV1 analysis +~~~~~~~~~~~~ + +AV1's current ``avd_av1_set_prob()`` reads the primary reference's CDF tail on +the CPU, and seeds the current frame's output CDF because hardware leaves it +untouched when DISABLE_FRAME_END_UPDATE_CDF is set. ``avd_av1_set_seg()`` can +also copy the primary reference's segmentation map on the CPU. These are +separate dependencies; removing only the CDF wait is insufficient. + +=================================== ========================================= +Case CDF source dependency +=================================== ========================================= +Intra/error-resilient/PRIMARY_NONE Defaults; no primary-reference CDF read +Primary ref with frame-end disabled Its initialized CDF, available at prepare +Primary ref with frame-end enabled Its hardware-written CDF tail +=================================== ========================================= + +``disable_frame_end_update_cdf`` on the *consumer* does not remove its primary +reference input dependency. It controls that consumer's future CDF output. +Likewise, a default CDF does not necessarily remove the segmentation-map input +selected by primary_ref_frame. + +The available implementation does not establish whether the CDF tail *and* +segmentation map are fully visible at VP DONE, or which CPU DMA synchronization +is required for an in-flight CAPTURE allocation while PP writes another part. +This differs from VP9's separate coherent count buffer. AV1 therefore retains +its one-picture path. Enabling it needs evidence for both tail writes and a +safe visibility/synchronization rule, then two scratch banks and a dependency +on the actual primary reference (which need not be the preceding picture). +No AV1 or VP9 pixel correctness or throughput claim follows from these host +and emulator tests; device qualification remains necessary. diff --git a/drivers/media/platform/apple/avd/tests/dispatch-test.py b/drivers/media/platform/apple/avd/tests/dispatch-test.py new file mode 100755 index 0000000000000..055f0d544bcae --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/dispatch-test.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise production submission/retirement with a strict fake m2m scheduler. + +The boundary rejects job_finish on HOLD_CAPTURE queues and a finish for any +context other than the running one. PM, work scheduling and codec preparation +are fixtures; avd_queue_submit_work(), avd_retire() and slot helpers are real. +""" +import os +from pathlib import Path +import hashlib +import subprocess +import sys + +HERE = Path(__file__).resolve().parent +ROOT = Path(sys.argv[2]).resolve() if len(sys.argv) > 2 else HERE.parent +CORE = Path(sys.argv[3]).resolve() if len(sys.argv) > 3 else ROOT.parents[4] / 'drivers/media/v4l2-core/v4l2-mem2mem.c' +from host_source import function + +PRELUDE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define AVD_H_ +#define AVD_QUEUE_H_ +#define AVD_MAX_FRAMES 16 +#define VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF 1 +#define V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF 1 +#define AVD_FRAME_TIMEOUT 2000 +#define AVD_RECOVER_GAP_MS 20 +#define msecs_to_jiffies(x) (x) +#define AVD_CTX_MAX_HW_ERRORS 1 +#define AVD_PROGRAM_SIZE 4096 +#define GFP_KERNEL 0 +#define dma_wmb() ((void)0) +#define dma_rmb() ((void)0) +#define system_dfl_wq NULL +#define time_before(a,b) ((long)((a)-(b))<0) +#define WRITE_ONCE(a,b) ((a)=(b)) +#define READ_ONCE(v) (v) +#define max(a,b) ((a)>(b)?(a):(b)) +static bool warn_on(bool v) { assert(!v);return v; } +#define WARN_ON(v) warn_on(!!(v)) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define spin_lock_irqsave(l,f) do { (void)(l); (f)=0; } while(0) +#define spin_unlock_irqrestore(l,f) do { (void)(l); (void)(f); } while(0) +#define mutex_lock(l) ((void)(l)) +#define mutex_unlock(l) ((void)(l)) +#define dev_err(d,...) ((void)(d)) +static unsigned long jiffies=1; +static void msleep(unsigned int ms) { assert(ms==20);jiffies+=ms; } +struct work_struct { int unused; }; +struct vb2_v4l2_buffer { unsigned int flags; bool is_held; int done; + struct { struct { void *req; } req_obj; } vb2_buf; +}; +struct avd_ctx; +struct avd_dev; +struct avd_buf { void *cpu; size_t size; }; +''' + +BOUNDARY = r''' +struct fake_m2m { + struct { struct { unsigned int subsystem_flags; } q; } out_q_ctx; + struct vb2_v4l2_buffer *src,*dst; + struct avd_ctx *ctx; + void *priv; +}; +#define v4l2_m2m_ctx fake_m2m +struct v4l2_m2m_dev { struct fake_m2m *curr_ctx; int job_spinlock; }; +static struct v4l2_m2m_dev core; +struct avd_codec_frame_ops { + int (*prepare)(struct avd_ctx *,struct avd_frame *); + void (*retire)(struct avd_ctx *,struct avd_frame *,enum vb2_buffer_state); +}; +struct avd_coded_fmt_ops { + bool (*ready)(struct avd_ctx *); + void (*vp_done)(struct avd_ctx *,struct vb2_v4l2_buffer *); + int (*run)(struct avd_ctx *); + void (*done)(struct avd_ctx *,struct vb2_v4l2_buffer *,struct vb2_v4l2_buffer *,enum vb2_buffer_state); +}; +struct format { const struct avd_codec_frame_ops *frame_ops; const struct avd_coded_fmt_ops *ops; }; +struct avd_ctx { + struct avd_dev *dev; struct { struct fake_m2m *m2m_ctx; } fh; + const struct format *coded_fmt_desc; struct work_struct submit_work; + bool draining,broken; unsigned int hw_errors,max_frames; enum avd_frame_dep next_dep; + bool session_started; u64 session_generation; + int ctrl_hdl; +}; +struct avd_variant { size_t fp_size; }; +struct avd_backend_ops { + int (*session_start)(struct avd_ctx *); + int (*submit)(struct avd_dev *,struct avd_frame *); +}; +struct avd_dev { + const struct avd_variant *variant; + const struct avd_backend_ops *backend; u64 fw_generation; + struct avd_frame frames[AVD_MAX_FRAMES]; struct avd_buf *frame_fp[AVD_MAX_FRAMES]; + struct avd_ctx *deferred_ctx; struct avd_frame *building; + unsigned int inflight; u64 frame_seq; bool compat_held,stopping,faulted; + unsigned long rest_until,last_recover; unsigned int storm_count; + int queue_lock,queue_mutex,queue_work; void *dev,*m2m_dev; +}; +static struct avd_ctx *running; +static unsigned int finishes,raw_finishes,pm_refs; +static int prepare_result,pm_error,send_result,wake_error; +static unsigned int wakes,irq_enables; +static int avd_pm_recover(struct avd_dev *d) { assert(pm_refs);assert(!d->last_recover||jiffies-d->last_recover>=20);wakes++;return wake_error; } +static void avd_pm_enable_irq(struct avd_dev *d) { (void)d;irq_enables++; } +static u32 send_tags[64];static unsigned int send_count; +static unsigned int capacity=1; +static struct vb2_v4l2_buffer *v4l2_m2m_next_src_buf(struct fake_m2m *m) { return m->src; } +static struct vb2_v4l2_buffer *v4l2_m2m_next_dst_buf(struct fake_m2m *m) { return m->dst; } +static struct vb2_v4l2_buffer *v4l2_m2m_src_buf_remove(struct fake_m2m *m) { + struct vb2_v4l2_buffer *b=m->src;m->src=NULL;return b; +} +static struct vb2_v4l2_buffer *v4l2_m2m_dst_buf_remove(struct fake_m2m *m) { + struct vb2_v4l2_buffer *b=m->dst;m->dst=NULL;return b; +} +static void finish(struct fake_m2m *m) { + assert(running==m->ctx&&core.curr_ctx==m);running=NULL;core.curr_ctx=NULL;finishes++; +} +static bool _v4l2_m2m_job_finish(struct v4l2_m2m_dev *d,struct fake_m2m *m) +{ assert(d==&core);finish(m);return true; } +static void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *d,struct fake_m2m *m) +{ (void)d;assert(!m->src&&(!m->dst||m->dst->is_held)); } +#define EXPORT_SYMBOL_GPL(f) +static void __attribute__((unused)) v4l2_m2m_job_finish(void *d,struct fake_m2m *m) { + (void)d; assert(!(m->out_q_ctx.q.subsystem_flags&VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF)); + raw_finishes++; finish(m); +} +static void v4l2_m2m_buf_done(struct vb2_v4l2_buffer *b,enum vb2_buffer_state s) +{ (void)s; assert(b && !b->done); b->done++; } +static void v4l2_m2m_buf_copy_metadata(void *s,void *d) { (void)s;(void)d; } +static void v4l2_ctrl_request_complete(void *r,void *h) { (void)r;(void)h; } +static int v4l2_ctrl_request_setup(void *r,void *h) { (void)r;(void)h; return 0; } +static unsigned int avd_max_inflight(struct avd_dev *d) { (void)d; return capacity; } +static bool avd_backend_is_apple(struct avd_dev *d) { (void)d; return false; } +static int pm_runtime_resume_and_get(void *d) { (void)d; if(pm_error)return pm_error; pm_refs++;return 0; } +static void pm_runtime_mark_last_busy(void *d) { (void)d; } +static void pm_runtime_put_autosuspend(void *d) { (void)d; assert(pm_refs);pm_refs--; } +static void avd_pm_queue_changed(struct avd_dev *d) { (void)d; } +static void mod_delayed_work(void *w,void *v,int t) { (void)w;(void)v;(void)t; } +static int avd_buf_alloc(struct avd_dev *d,struct avd_buf *b,size_t n) { (void)d;(void)b;(void)n;return -ENOMEM; } +static void *devm_kzalloc(void *d,size_t n,int f) { (void)d;(void)f;return calloc(1,n); } +static int transport_submit(struct avd_dev *d,struct avd_frame *f) { + (void)d;if(!send_result)send_tags[send_count++]=f->tag;return send_result; +} +static int codec_prepare(struct avd_ctx *c,struct avd_frame *f) { (void)c;(void)f;return prepare_result; } +void avd_job_finish(struct avd_ctx *c,enum vb2_buffer_state state); +static int codec_run(struct avd_ctx *c) { + if(!prepare_result&&(c->fh.m2m_ctx->src->flags&V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) + avd_job_finish(c,VB2_BUF_STATE_DONE); + return prepare_result; +} +''' + +MAIN = r''' +static void submit(struct avd_ctx *c) { assert(!running);running=c;core.curr_ctx=c->fh.m2m_ctx;avd_queue_submit_work(&c->submit_work); } +static struct avd_frame *owned(struct avd_dev *d,struct avd_ctx *c) { + for(unsigned int i=0;i<16;i++)if(d->frames[i].ctx==c&&d->frames[i].state!=AVD_FRAME_FREE)return &d->frames[i]; + return NULL; +} +static void complete(struct avd_dev *d,struct avd_ctx *c,int status) { + struct avd_frame *f=owned(d,c);assert(f);f->state=AVD_FRAME_DONE;f->status=status;avd_retire(d); +} +int main(void) { + static const struct avd_backend_ops backend={.submit=transport_submit}; + struct avd_dev d={.m2m_dev=&core,.backend=&backend};struct avd_buf programs[16]={}; + static const struct avd_coded_fmt_ops ops={.run=codec_run}; + static const struct format fmt={.ops=&ops}; + struct fake_m2m m[2]={};struct avd_ctx c[2]={}; + struct vb2_v4l2_buffer src[24]={},dst[24]={}; + for(unsigned int i=0;i<16;i++){d.frames[i].idx=i;d.frames[i].prog=&programs[i];} + for(unsigned int i=0;i<2;i++){c[i].dev=&d;c[i].fh.m2m_ctx=&m[i];c[i].coded_fmt_desc=&fmt;m[i].ctx=&c[i];m[i].priv=&c[i];} + /* Non-HOLD jobs detach once, finish dispatch once, retire buffers later. */ + m[0].src=&src[0];m[0].dst=&dst[0];submit(&c[0]);assert(!running&&d.inflight==1&&finishes==1); + assert(!src[0].done&&!dst[0].done); + /* The next context keeps the running dispatch while capacity is full. */ + m[1].src=&src[1];m[1].dst=&dst[1];submit(&c[1]); + assert(running==&c[1]&&d.deferred_ctx==&c[1]&&finishes==1); + complete(&d,&c[0],0);assert(src[0].done&&dst[0].done&&!d.inflight); + d.deferred_ctx=NULL;avd_queue_submit_work(&c[1].submit_work); + assert(!running&&finishes==2);complete(&d,&c[1],0);assert(finishes==2); + /* Final slices release dispatch early without returning either DMA buffer. */ + m[0].out_q_ctx.q.subsystem_flags=VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF; + m[0].src=&src[2];m[0].dst=&dst[2];submit(&c[0]); + assert(!running&&finishes==3&&!owned(&d,&c[0])->scheduler_held); + assert(!m[0].src&&!m[0].dst&&!src[2].done&&!dst[2].done); + complete(&d,&c[0],0);assert(finishes==3&&!raw_finishes); + /* A held intermediate slice returns only OUTPUT and keeps CAPTURE. */ + src[3].flags=V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF; + m[0].src=&src[3];m[0].dst=&dst[3];submit(&c[0]); + assert(!d.inflight&&!running&&src[3].done&&!dst[3].done&&dst[3].is_held); + src[13].flags=V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;m[0].src=&src[13];submit(&c[0]); + assert(!d.inflight&&!running&&src[13].done&&!dst[3].done&&dst[3].is_held); + m[0].src=&src[4];submit(&c[0]); + assert(!running&&!src[4].done&&!dst[3].done&&!dst[3].is_held); + complete(&d,&c[0],0);assert(src[4].done&&dst[3].done); + /* Final-slice prepare failures detach early and retire buffers with error. */ + prepare_result=-EINVAL;m[0].src=&src[5];m[0].dst=&dst[5];submit(&c[0]); + assert(!running&&!src[5].done&&!dst[5].done);avd_retire(&d); + assert(src[5].done&&dst[5].done); + c[0].broken=false;prepare_result=0; + /* PM failure and stopped-stream dispatch finish the current job once. */ + pm_error=-EIO;m[0].src=&src[6];m[0].dst=&dst[6];submit(&c[0]);assert(!running&&!d.inflight); + pm_error=0;c[0].draining=true;m[0].src=&src[7];m[0].dst=&dst[7];submit(&c[0]); + assert(!running&&src[7].done&&dst[7].done&&!d.inflight&&!pm_refs); + /* The native prepare failure path detaches and finishes a non-HOLD job. */ + static const struct avd_codec_frame_ops native_ops={.prepare=codec_prepare}; + static const struct format native_fmt={.frame_ops=&native_ops}; + c[1].coded_fmt_desc=&native_fmt;prepare_result=-EINVAL; + programs[0].cpu=&programs[0];m[1].src=&src[8];m[1].dst=&dst[8]; + submit(&c[1]);assert(!running);avd_retire(&d); + assert(src[8].done&&dst[8].done&&!d.inflight&&!pm_refs); + /* The next picture really prepares while the first still owns DMA buffers. */ + c[0].draining=false;c[0].broken=false;prepare_result=0;capacity=2; + m[0].src=&src[9];m[0].dst=&dst[9];submit(&c[0]); + m[0].src=&src[10];m[0].dst=&dst[10];submit(&c[0]); + assert(d.inflight==2&&!running&&!src[9].done&&!dst[9].done&&!src[10].done&&!dst[10].done); + complete(&d,&c[0],0);assert(src[9].done&&dst[9].done&&!src[10].done&&!dst[10].done); + complete(&d,&c[0],0);assert(src[10].done&&dst[10].done&&!d.inflight&&!pm_refs&&!raw_finishes); + unsigned int before=finishes; + avd_queue_submit_work(&c[0].submit_work);assert(finishes==before&&!d.inflight); + capacity=1;c[1].coded_fmt_desc=&fmt;c[1].broken=false; + m[0].src=&src[11];m[0].dst=&dst[11];submit(&c[0]); + m[1].src=&src[12];m[1].dst=&dst[12];submit(&c[1]); + assert(running==&c[1]&&d.deferred_ctx==&c[1]);before=finishes; + avd_queue_submit_work(&c[0].submit_work); + assert(running==&c[1]&&d.deferred_ctx==&c[1]&&finishes==before); + complete(&d,&c[0],0);avd_queue_submit_work(&c[1].submit_work);complete(&d,&c[1],0); + assert(!running&&!d.inflight&&!pm_refs&&!raw_finishes); + /* Inbox pressure cannot reverse two pictures of a context. */ + capacity=3;send_result=-EBUSY;unsigned int sent=send_count; + m[0].src=&src[14];m[0].dst=&dst[14];submit(&c[0]); + u32 first=d.frames[0].tag; + assert(d.frames[0].state==AVD_FRAME_RESERVED&&send_count==sent); + send_result=0;m[0].src=&src[15];m[0].dst=&dst[15];submit(&c[0]); + u32 second=d.frames[1].tag; + assert(d.frames[1].state==AVD_FRAME_RESERVED&&send_count==sent); + m[1].src=&src[16];m[1].dst=&dst[16];submit(&c[1]); + assert(d.frames[2].state==AVD_FRAME_SUBMITTED&&send_count==sent+1); + assert(!avd_send_frame(&d,&d.frames[0]));assert(!avd_send_frame(&d,&d.frames[1])); + assert(send_tags[sent+1]==first&&send_tags[sent+2]==second); + complete(&d,&c[1],0);complete(&d,&c[0],0);complete(&d,&c[0],0); + assert(!running&&!d.inflight&&!pm_refs&&!raw_finishes); + /* Rest rejects new frames without taking PM refs or waking hardware. */ + d.rest_until=jiffies+100;unsigned int old_wakes=wakes; + m[0].src=&src[17];m[0].dst=&dst[17];submit(&c[0]); + assert(!running&&!d.inflight&&!pm_refs&&wakes==old_wakes&&src[17].done&&dst[17].done); + jiffies=d.rest_until;d.storm_count=3;c[0].broken=false; + m[0].src=&src[18];m[0].dst=&dst[18];submit(&c[0]); + assert(!running&&wakes==old_wakes+1&&irq_enables==1&&!d.rest_until&&!d.storm_count); + complete(&d,&c[0],0);assert(!pm_refs); + /* A failed wake retires the decoder and never opens IRQs or posts a frame. */ + d.rest_until=jiffies;wake_error=-EIO;m[0].src=&src[19];m[0].dst=&dst[19]; + unsigned int old_sent=send_count;submit(&c[0]);assert(d.faulted&&irq_enables==1&&send_count==old_sent); + avd_retire(&d);assert(!running&&!pm_refs&&src[19].done&&dst[19].done); + puts("PASS: production dispatch/retire: rest/wake, final/held slices, two contexts, capacity retry, prepare/PM/abort errors, exactly-once job_finish"); +} +''' + +VP_MAIN = r''' +static unsigned int vp_calls; +static bool dependency_ready; +static bool fake_ready(struct avd_ctx *c) { (void)c;return dependency_ready; } +static void vp_callback(struct avd_ctx *c,struct vb2_v4l2_buffer *dst) { + (void)c;assert(dst&&!dst->done);vp_calls++;dependency_ready=true; +} +static void check_vp_events(void) { + struct avd_dev d={};struct avd_ctx c={.dev=&d,.next_dep=AVD_DEP_PREV_DONE,.max_frames=2}; + struct vb2_v4l2_buffer dst={}; + const struct avd_coded_fmt_ops ops={.ready=fake_ready,.vp_done=vp_callback}; + const struct format fmt={.ops=&ops};c.coded_fmt_desc=&fmt; + struct avd_frame *f=&d.frames[0];f->idx=0; + assert(!avd_slot_reserve(f,&c,1));f->state=AVD_FRAME_SUBMITTED;f->dst=&dst; + assert(!avd_queue_ready(&c)); + avd_frame_vp_complete(&d,0x10010);assert(!f->vp_done&&!vp_calls); + avd_frame_vp_complete(&d,0x10);assert(f->vp_done&&!vp_calls&&!f->vp_retired); + assert(!avd_queue_ready(&c));avd_retire_vp(&d); + assert(vp_calls==1&&f->vp_retired&&avd_queue_ready(&c)); + assert(f->state==AVD_FRAME_SUBMITTED&&f->dst==&dst&&!dst.done); + avd_frame_vp_complete(&d,0x10);avd_retire_vp(&d);assert(vp_calls==1); + f->state=AVD_FRAME_FREE;assert(!avd_slot_reserve(f,&c,2));assert(!f->vp_done&&!f->vp_retired); + f->state=AVD_FRAME_SUBMITTED;avd_frame_vp_complete(&d,0x10);assert(!f->vp_done); + avd_frame_vp_complete(&d,0x20);f->status=-EIO;f->state=AVD_FRAME_DONE; + avd_retire_vp(&d);assert(vp_calls==1&&f->vp_retired); + puts("PASS: production VP event: stale/duplicate/reset tags, callback dependency and retained PP ownership"); +} +''' + + +def main(): + out = Path(sys.argv[1]).resolve() + out.mkdir(parents=True, exist_ok=True) + names = ['avd-queue.c', 'avd-queue-internal.h', 'avd-queue.h', 'avd-v4l2.c'] + (out / 'core.sha256').write_text(hashlib.sha256(CORE.read_bytes()).hexdigest()+' v4l2-mem2mem.c\n') + (out / 'inputs.sha256').write_text(''.join(hashlib.sha256((ROOT / n).read_bytes()).hexdigest()+' '+n+'\n' for n in names)) + qh = (ROOT / 'avd-queue.h').read_text() + frame = qh[qh.index('enum avd_frame_state'):qh.index('struct avd_backend_ops {')] + source = (ROOT / 'avd-queue.c').read_text() + legacy = (ROOT / 'avd-v4l2.c').read_text() + code = out / 'dispatch.c' + code.write_text(PRELUDE + frame + BOUNDARY + '\n#include "avd-queue-internal.h"\n' + + function(CORE.read_text(), 'v4l2_m2m_get_curr_priv') + + function(CORE.read_text(), 'v4l2_m2m_job_finish_detach') + + function(CORE.read_text(), 'v4l2_m2m_buf_done_and_job_finish') + + function(legacy, 'avd_job_finish_no_pm') + function(legacy, 'avd_job_finish') + + function(source, 'avd_send_frame') + + function(source, 'avd_retire') + function(source, 'avd_queue_submit_work') + + function(source, 'avd_frame_vp_complete') + function(source, 'avd_retire_vp') + + function(source, 'avd_queue_ready') + VP_MAIN + MAIN.replace(' puts("PASS: production dispatch', ' check_vp_events();\n puts("PASS: production dispatch')) + exe = out / 'dispatch-test' + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', '-g', '-I' + str(HERE / 'include'), + '-I' + str(ROOT), str(code), '-o', str(exe)], check=True) + subprocess.run([str(exe)], check=True) + + +if __name__ == '__main__': + main() diff --git a/drivers/media/platform/apple/avd/tests/host_source.py b/drivers/media/platform/apple/avd/tests/host_source.py new file mode 100644 index 0000000000000..b473b65e3c4ed --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/host_source.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Extract unchanged production function bodies for host boundary tests.""" +import re + + +def function(source, name): + match = re.search(r"^[A-Za-z_][\w *]*\b" + re.escape(name) + + r"\([^;{}]*?\)\n\{", source, re.M) + if not match: + raise ValueError(name) + start = source.index("{", match.start()) + depth = 1 + end = start + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + "\n" diff --git a/drivers/media/platform/apple/avd/tests/include/linux/errno.h b/drivers/media/platform/apple/avd/tests/include/linux/errno.h new file mode 100644 index 0000000000000..32a2df3ba8d20 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/errno.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include_next diff --git a/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h new file mode 100644 index 0000000000000..ccda48a6de34b --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_INTERRUPT_H_ +#define AVD_HOST_INTERRUPT_H_ +#define irqreturn_t int +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h b/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h new file mode 100644 index 0000000000000..a73bb63db2c8e --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_JIFFIES_H_ +#define AVD_HOST_JIFFIES_H_ +#include +#define time_after_eq(a, b) ((long)((a) - (b)) >= 0) +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/limits.h b/drivers/media/platform/apple/avd/tests/include/linux/limits.h new file mode 100644 index 0000000000000..304aeeeff0fbd --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/limits.h @@ -0,0 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#define U32_MAX UINT32_MAX diff --git a/drivers/media/platform/apple/avd/tests/include/linux/types.h b/drivers/media/platform/apple/avd/tests/include/linux/types.h new file mode 100644 index 0000000000000..8b9638abb9a7f --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/types.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_TYPES_H_ +#define AVD_HOST_TYPES_H_ +#include +#include +#include_next +typedef __u8 u8; +typedef __u32 u32; +typedef __u64 u64; +typedef __u64 dma_addr_t; +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h b/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h new file mode 100644 index 0000000000000..6cd054cc198d3 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_VB2_H_ +#define AVD_HOST_VB2_H_ +enum vb2_buffer_state { VB2_BUF_STATE_DONE, VB2_BUF_STATE_ERROR }; +#endif diff --git a/drivers/media/platform/apple/avd/tests/program-test.py b/drivers/media/platform/apple/avd/tests/program-test.py new file mode 100755 index 0000000000000..ad5d2ec009d95 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/program-test.py @@ -0,0 +1,385 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production H.264 emitters and open-program builder on a host. + +Only allocation, controls, and buffer addresses are fixtures. No copied hardware +program, firmware payload, or device capture is used. Requires Linux UAPI headers. +The output is a codec-generated 1920x1088 IDR with two I slices, not a +pixel decoder. Run in a build slot: program-test.py OUTPUT_DIRECTORY. +""" +import os +import hashlib +from pathlib import Path +import re +import signal +import subprocess +import sys + +from host_source import function + +ROOT = Path(sys.argv[2]).resolve() if len(sys.argv) > 2 else Path(__file__).resolve().parent.parent + + + + +PRELUDE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef int32_t s32; +typedef uint64_t u64; +typedef uint64_t dma_addr_t; +#define AVD_H_ +#define BIT(n) (1U << (n)) +#define GENMASK(h,l) ((UINT32_MAX >> (31-(h))) & (UINT32_MAX << (l))) +#define FIELD_PREP(m,v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#define DIV_ROUND_UP(n,d) (((n)+(d)-1)/(d)) +#define VISIBLE_IF_KUNIT +#define AVD_QUIRK_LSR 1 +#define AVD_QUIRK_NO_PIPE_STATE 2 +#define AVD_CODEC_H264 1 +#define AVD_CODEC_VP9 2 +#define AVD_MAX_INST 512 +#define AVD_CODEC_AV1 3 +#define AVD_MAX_FRAMES 16 +#define AVD_IMG_FMT_420_10BIT 1 +#define AVD_IMG_FMT_422_10BIT 2 +#define AVD_FRAME_SUBMITTED 2 +#define AVD_FRAME_DONE 3 +#define IRQ_HANDLED 1 +#define system_dfl_wq NULL +typedef int irqreturn_t; +#define dev_info_once(d,...) ((void)(d)) +#define dev_warn_once(d,...) ((void)(d)) +#define dev_warn_ratelimited(d,...) ((void)(d)) +#define __iomem +#define USEC_PER_MSEC 1000 +#define min_t(t,a,b) ((t)(a)<(t)(b)?(t)(a):(t)(b)) +#define min3(a,b,c) ((a)<(b) ? ((a)<(c)?(a):(c)) : ((b)<(c)?(b):(c))) +#define wmb() ((void)0) +#define dev_dbg(d,...) ((void)(d)) +#define dev_err_ratelimited(d,...) ((void)(d)) +static unsigned long jiffies; +static unsigned long usecs_to_jiffies(unsigned long t) { return t/1000; } +static bool time_after(unsigned long a,unsigned long b) { return (long)(b-a)<0; } +static void usleep_range(unsigned int a,unsigned int b) { (void)a;(void)b;jiffies++; } +static u32 readl(void *p) { assert(!((uintptr_t)p&3));return *(u32 *)p; } +static void writel(u32 v,void *p) { assert(!((uintptr_t)p&3));*(u32 *)p=v; } +#define writel_relaxed writel +static void __iowrite32_copy(void *p,const void *s,size_t n) { + const u32 *src=s;for(size_t i=0;i>shift) && dma_countjob.num,ctx->job.segments[ctx->job.num].num,shift,width,addr}; +} +static bool avd_backend_is_qfw(struct avd_dev *d) { return d->queued; } +static dma_addr_t next_iova=0x204000000ULL; +static int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n) +{ (void)d; free(b->cpu); b->cpu = calloc(1,n); b->size = n; + b->addr=next_iova; next_iova+=(n+0xffff)&~0xffffULL; return b->cpu ? 0 : -ENOMEM; } +static void avd_buf_free(struct avd_dev *d, struct avd_buf *b) { (void)d;free(b->cpu);memset(b,0,sizeof(*b)); } +static struct avd_decoded_buffer *vb2_to_avd_decoded_buf(struct vb2_buffer *b) +{ return (void *)b; } +static struct avd_decoded_buffer reference; +static struct avd_decoded_buffer *avd_get_ref_buf(struct avd_ctx *c, + struct vb2_v4l2_buffer *b, u64 t) { (void)c; (void)b; (void)t; return &reference; } +static dma_addr_t vb2_dma_contig_plane_dma_addr(struct vb2_buffer *b, int p) +{ (void)p; return b->dma; } +static struct vb2_buffer *vb2_find_buffer(void *q, u64 t) +{ (void)q; (void)t; return NULL; } +static unsigned int fmt_width(struct avd_ctx *c) { (void)c; return 1920; } +static unsigned int fmt_height(struct avd_ctx *c) { (void)c; return 1088; } +static void avd_qfw_debug(struct avd_dev *d,struct avd_frame *f,const char *w) { (void)d;(void)f;(void)w; } +static unsigned int pp_kicks(struct avd_ctx *c) { return c->pp_total; } +static u32 messages[16];static unsigned int msg_read,msg_count,completed,reset_requested; +static bool avd_mbox_pop(struct avd_dev *d,u32 *word) { + (void)d;if(msg_read==msg_count)return false;*word=messages[msg_read++];return true; +} +static unsigned int vp_completed; +static void avd_frame_vp_complete(struct avd_dev *d,u32 tag) { assert(d->frames[tag&15].tag==tag);vp_completed++; } +static void avd_frame_complete(struct avd_dev *d,u32 tag,int status) { + struct avd_frame *f=&d->frames[tag&15];assert(f->tag==tag&&f->state==AVD_FRAME_SUBMITTED); + f->state=AVD_FRAME_DONE;f->status=status;completed++; +} +static void mod_delayed_work(void *w,void *v,int t) { (void)w;(void)v;(void)t;reset_requested++; } +''' + +MAIN = r''' +int main(int argc, char **argv) +{ + struct avd_variant variant = { .revision=4, .quirks=AVD_QUIRK_LSR, + .fifo_slots=15, .vp_slots={4,4,1,0}, .configure_stream=t8112_configure_stream }; + struct avd_buf program = {}; + struct avd_frame frame = { .prog=&program }; + struct avd_dev dev = { .variant=&variant, .building=&frame, .queued=true }; + struct avd_ctx ctx = { .dev=&dev, .job.codec=AVD_CODEC_H264 }; + typeof(*ctx.fh.m2m_ctx) m2m = {}; + struct avd_h264_ctx h264[8] = {}; + struct avd_qfw q = { .sram=calloc(1,0x10000), .aperture=0x1090000, + .ring_words=0x1800, .fifo_slots=15 }; + static const struct avd_coded_fmt_ops ops={.pp_kicks=pp_kicks}; + static const struct fmt_desc fmt={.ops=&ops}; + dev.qfw=&q;dev.ctrl=calloc(1,0x10000);dev.mbox=calloc(1,0x100);dev.capabilities=7; + ctx.coded_fmt_desc=&fmt;frame.ctx=&ctx;ctx.fh.m2m_ctx=&m2m; + /* Optional VP notifications are enabled only by the matching feature bit. */ + u32 info[16]={AVDQ_MAGIC,2,16,64,0x1090000,0x8000,256,0,9,7,15,3,0,0x1800,0,0}; + memcpy(q.sram+AVDQ_INFO,info,sizeof(info));assert(avd_qfw_check(&dev)&&!q.vp_done); + info[11]=7;memcpy(q.sram+AVDQ_INFO,info,sizeof(info));assert(avd_qfw_check(&dev)&&q.vp_done); + info[1]=3;memcpy(q.sram+AVDQ_INFO,info,sizeof(info));assert(!avd_qfw_check(&dev)&&!q.vp_done); + memset(q.sram,0,0x10000); + struct avd_buf slice = { .size=4096, .addr=0x200800000ULL }; + struct v4l2_ctrl_h264_sps sps = { .pic_width_in_mbs_minus1=119, + .pic_height_in_map_units_minus1=67, .chroma_format_idc=1, + .flags=V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY | V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE }; + struct v4l2_ctrl_h264_decode_params decode = { .flags=V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC }; + struct v4l2_ctrl_h264_pps pps = {}; + struct v4l2_ctrl_h264_pred_weights weights = {}; + struct avd_decoded_buffer dst = { .base.vb.vb2_buf.dma=0x203000000ULL }; + struct v4l2_ctrl_h264_slice_params sl = { .slice_type=V4L2_H264_SLICE_TYPE_I, .header_bit_size=32 }; + struct avd_h264_run run = { .sps=&sps, .pps=&pps, .decode_params=&decode, + .slice_params=&sl, .slice_data_off=4, .pred_weights=&weights, + .addresses.mv_color=0x203800000ULL, .num_valid=1, + .base={ .comp_out=0x200100000ULL, .y_out=0x200200000ULL, .uv_out=0x200400000ULL } }; + assert(argc==3); + bool v5=!strncmp(argv[2],"v5",2); + if (v5) { + variant.quirks=0;variant.configure_stream=t8122_configure_stream; + if (!strcmp(argv[2],"v5-t0")) { variant.fifo_slots=7;variant.vp_slot_base[1]=4; } + } else if (!strcmp(argv[2],"v3-t0")) variant.quirks|=AVD_QUIRK_NO_PIPE_STATE; + q.fifo_slots=variant.fifo_slots; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; + run.base.bufs.dst=&dst.base.vb; + for(unsigned int c=0;c<8;c++) { + ctx.priv=&h264[c];h264[c].nbanks=AVD_H264_BANKS;h264[c].b=&h264[c].bank[0]; + for(unsigned int b=0;bb==&hc->bank[idx%AVD_H264_BANKS]);hc->b->active_slice=&slice; + hc->b->submitted=true; + ctx.job_fifo=hc->b->bufs.inst.addr; + assert(ctx.job_fifo && hc->b->bufs.above_info.addr && hc->b->bufs.ip_above.addr && + hc->b->bufs.lf_above.addr && hc->b->bufs.lf_above_info.addr && hc->b->bufs.mv_above_info.addr); + assert((variant.quirks&AVD_QUIRK_NO_PIPE_STATE)||hc->b->bufs.pipe_state.addr); + frame.idx=idx; ctx.job.num=0;ctx.pp_total=1; + dma_count=0; + run.base.comp_out=0x230000000ULL+idx*0x1000000ULL; + run.base.y_out=run.base.comp_out+0x100000; + run.base.uv_out=run.base.comp_out+0x300000; + run.addresses.mv_color=run.base.y_out+0x700000; + reference.base.vb.vb2_buf.dma=idx?run.base.comp_out-0x1000000:run.base.comp_out; + decode.flags=idx%3?0:V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC; + decode.dpb[0].flags=idx%3?V4L2_H264_DPB_ENTRY_FLAG_VALID:0; + sl.slice_type=idx%3==0?V4L2_H264_SLICE_TYPE_I:idx%3==1?V4L2_H264_SLICE_TYPE_P:V4L2_H264_SLICE_TYPE_B; + ctx.job.segments=calloc(3,sizeof(*ctx.job.segments)); assert(ctx.job.segments); + stream_hdr(&ctx,&run); ctx.job.num=1;sl.first_mb_in_slice=0;slice.addr=0x200800000ULL; + stream_slice(&ctx,&run); + size_t continuation=1+ctx.job.segments[0].num+ctx.job.segments[1].num; + ctx.job.num=2;sl.first_mb_in_slice=4080;slice.addr+=0x1000; + stream_slice(&ctx,&run); + size_t starts[3]={1,1+ctx.job.segments[0].num,2+ctx.job.segments[0].num+ctx.job.segments[1].num}; + assert(!avd_open_build_program(&ctx)); + u32 fifo=dev.queued ? idx<<4 : 0; + assert(((u32 *)program.cpu)[0]==(0x2b000200U|fifo)); + assert(((u32 *)program.cpu)[continuation]==(0x2b000000U|fifo)); + assert(((u32 *)program.cpu)[frame.prog_words-1]==(0x2b000400U|fifo)); + if (dev.queued) { + frame.tag=0x10|idx;assert(frame.fifo_iova==hc->b->bufs.inst.addr && !ctx.job_fifo); + q.wp=0;memset(q.sram,0,0x10000);memset(dev.ctrl,0,0x10000); + writel(1,dev.mbox+0x50); + assert(avd_qfw_submit(&dev,&frame)==-EBUSY); + unsigned int lo=v5?0x1d0:0x150; + assert(!readl(q.sram+0x8000)&&!q.wp&&!readl(dev.ctrl+lo+idx*4)); + for(unsigned int other=0;other>8)); + assert(readl(dev.ctrl+(v5?0x16c:0x10c))==7&&readl(dev.ctrl+(v5?0x190:0x120))==5); + snprintf(path,sizeof(path),"%s/mmio-%u.bin",argv[1],idx); + out=fopen(path,"wb");assert(out);assert(fwrite(dev.ctrl,0x10000,1,out)==1);assert(!fclose(out)); + snprintf(path,sizeof(path),"%s/descriptor-%u.bin",argv[1],idx); + out=fopen(path,"wb");assert(out);assert(fwrite(q.sram+0x8000+idx*64,64,1,out)==1);assert(!fclose(out)); + snprintf(path,sizeof(path),"%s/dma-sites-%u.txt",argv[1],idx); + out=fopen(path,"w");assert(out); + for(unsigned int a=0;asegment]+site->word,site->width,site->shift,(unsigned long long)site->addr); + } + assert(!fclose(out)); + } + snprintf(path,sizeof(path),"%s/%s-%u.bin",argv[1],dev.queued?"h264":"legacy",idx); + out=fopen(path,"wb"); assert(out); + /* Explicit LE wire format, independent of the host byte order. */ + for (size_t i=0; i>(8*b))&255,out); + } + assert(!fclose(out)); + } + } + /* Both ring-position wrap and u32-counter wrap must split the I/O copy. */ + for (unsigned int test=0;test<2;test++) { + u32 words[32];u32 start=test?UINT32_MAX-10:0x1800-8; + for(unsigned int i=0;i<32;i++)words[i]=0x12340000U+i; + memset(q.sram,0,0x10000);q.wp=start;writel(start,q.sram+0x7ff8); + assert(!avd_qfw_stream(&dev,words,32)); + assert(q.wp==(u32)(start+32)); + for(unsigned int i=0;i<32;i++)assert(readl(q.sram+4*((u32)(start+i)%0x1800))==words[i]); + } + /* A sole fatal completion must keep DMA buffers owned until reset. */ + memset(q.sram,0,0x10000);jiffies=123; + dev.frames[0].tag=0x10;dev.frames[0].state=AVD_FRAME_SUBMITTED; + void *desc=q.sram+0x8000; + writel(2,desc+D_STATE);writel(2,desc+D_STATUS);writel(0x10,desc+D_CTAG); + messages[0]=0x54530010;msg_count=1;msg_read=0; + assert(avd_qfw_irq(&dev)==IRQ_HANDLED); + assert(!completed&&reset_requested&&dev.frames[0].state==AVD_FRAME_SUBMITTED); + assert(dev.frames[0].deadline==jiffies); + /* Drain both valid tagged words and reject a stale full tag. */ + dev.frames[1].tag=0x11;dev.frames[1].state=AVD_FRAME_SUBMITTED; + writel(0,desc+D_STATUS); + writel(2,desc+64+D_STATE);writel(0x11,desc+64+D_CTAG); + messages[1]=0x54530011;msg_count=2;msg_read=0; + avd_qfw_irq(&dev);assert(completed==2&&msg_read==2); + dev.frames[0].state=AVD_FRAME_SUBMITTED;dev.frames[0].tag=0x20; + writel(2,desc+D_STATE);messages[0]=0x54530010;msg_count=1;msg_read=0; + avd_qfw_irq(&dev);assert(completed==2&&dev.frames[0].state==AVD_FRAME_SUBMITTED); + /* A missing saved FIFO must fail before any ownership/MMIO changes, even + * when the mutable context happens to contain another nonzero address. */ + dev.queued=true;frame.fifo_iova=0;ctx.inst.addr=0x208000000ULL; + writel(0x20001,dev.mbox+0x50);u32 old_wp=q.wp; + for(unsigned int idx=0;idxinst.cpu);free(buf->pipe_state.cpu);free(buf->above_info.cpu); + free(buf->lf_above_info.cpu);free(buf->lf_above.cpu);free(buf->ip_above.cpu);free(buf->mv_above_info.cpu); + } + free(program.cpu);free(q.sram);free(dev.ctrl);free(dev.mbox); + puts("PASS: production H.264 banks, IDR/P/B DMA operands, saved FIFO addresses and legacy EXECs"); +} +''' + + +def main(): + out = Path(sys.argv[1]).resolve() + out.mkdir(parents=True, exist_ok=True) + names = ['avd-h264.c', 'avd-inst.h', 'avd-backend-open.c', 'avd-backend-queue.c', 'avd-hw.c'] + (out / 'inputs.sha256').write_text(''.join(hashlib.sha256((ROOT / n).read_bytes()).hexdigest()+' '+n+'\n' for n in names)) + h264 = (ROOT / 'avd-h264.c').read_text().split('static int avd_h264_alloc_bufs(')[0] + h264 = re.sub(r'^#include.*\n', '', h264, flags=re.M) + inst = (ROOT / 'avd-inst.h').read_text() + inst = re.sub(r'^#include.*\n', '', inst, flags=re.M) + # Use the actual codec push/address/comp helpers, not fixture emitters. + builder = function((ROOT / 'avd-backend-open.c').read_text(), 'avd_open_build_program') + code = out / 'program.c' + qfw = (ROOT / 'avd-backend-queue.c').read_text() + wire = qfw[qfw.index('#define AVDQ_MAGIC'):qfw.index('/* Arguments of dev_dbg()')] + hw = (ROOT / 'avd-hw.c').read_text() + regs = hw[hw.index('#define AVD_V3_'):hw.index('int avd_boot(')] + config = '#define w32(r,v) writel((v),avd->ctrl+(r))\n#define m32(r,v) w32(r,(v)|readl(avd->ctrl+(r)))\n' + config += function(hw, 't8112_configure_stream') + function(hw, 't8122_configure_stream') + # Address emitters must never consume a zero bank/output/reference address. + inst = inst.replace('static inline void push_address(struct avd_ctx *ctx, dma_addr_t addr)\n{', + 'static inline void push_address(struct avd_ctx *ctx, dma_addr_t addr)\n{\n record_dma(ctx,addr,(ctx->dev->variant->quirks&AVD_QUIRK_LSR)?8:0,(ctx->dev->variant->quirks&AVD_QUIRK_LSR)?1:2);') + inst = inst.replace('u32 offsets[4])\n{', 'u32 offsets[4])\n{\n assert(addr);') + inst = inst.replace('push(ctx, (addr + offsets[i]) >> 7);', '{ record_dma(ctx,addr+offsets[i],7,1); push(ctx, (addr + offsets[i]) >> 7); }') + code.write_text(PRELUDE + inst + h264 + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_alloc_bufs') + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_drop_bank_slices') + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_pick_bank') + builder + wire + regs + config + + function(qfw, 'avd_qfw_check') + function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') + + function(qfw, 'avd_qfw_fail_all') + function(qfw, 'avd_qfw_irq') + MAIN) + exe = out / 'program-test' + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', + '-Werror', '-Wno-sign-compare', '-Wno-unused-function', + '-fsanitize=address,undefined', '-g', str(code), '-o', str(exe)], check=True) + for variant in ('v3-t0', 'v3-t1', 'v5-t0', 'v5-t2'): + target = out / variant + target.mkdir(exist_ok=True) + subprocess.run([str(exe), str(target), variant], check=True) + for field in ('inst', 'pipe_state', 'above_info', 'ip_above', 'lf_above', 'lf_above_info', 'mv_above_info'): + target = out / ('zero-' + field) + target.mkdir(exist_ok=True) + with (target / 'expected-failure.log').open('w') as log: + result = subprocess.run([str(exe), str(target), 'v3-t1'], + env=dict(os.environ, AVD_ZERO_BANK_ADDR=field), + stdout=log, stderr=subprocess.STDOUT) + assert result.returncode == -signal.SIGABRT, (field, result.returncode) + print('PASS: host test rejects zero bank-1 ' + field, flush=True) + + +if __name__ == '__main__': + main() diff --git a/drivers/media/platform/apple/avd/tests/queue-host.c b/drivers/media/platform/apple/avd/tests/queue-host.c new file mode 100644 index 0000000000000..ac47decffe3cf --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/queue-host.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the production helpers with deterministic shuffled completions. */ +#include +#include +#include + +#include "../avd-apple-protocol.h" +#include "../avd-queue-internal.h" + +int main(void) +{ + struct avd_frame pool[AVD_MAX_FRAMES] = {}; + int clients[3]; + u64 last[3] = {}, seq = 0; + u32 rng = 0x3851, old_tag[AVD_MAX_FRAMES] = {}; + unsigned int i, step, completed = 0; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + pool[i].idx = i; + for (step = 0; step < 100000; step++) { + struct avd_frame *f; + + rng = rng * 1664525 + 1013904223; + f = &pool[(rng >> 16) & 15]; + if (f->state == AVD_FRAME_FREE) { + struct avd_ctx *ctx = (void *)&clients[(rng >> 8) % 3]; + + assert(!avd_slot_reserve(f, ctx, ++seq)); + assert(!avd_slot_complete(f, f->tag, 0)); + f->deadline = step + 100; + f->state = AVD_FRAME_SUBMITTED; + assert(!avd_slot_complete(f, old_tag[f->idx], 0)); + old_tag[f->idx] = f->tag; + } else if (f->state == AVD_FRAME_SUBMITTED) { + assert(avd_slot_complete(f, f->tag, (rng & 1) ? -EIO : 0)); + assert(!avd_slot_complete(f, f->tag, 0)); + } + for (i = 0; i < AVD_MAX_FRAMES; i++) { + unsigned int c; + + f = &pool[i]; + if (f->state != AVD_FRAME_DONE || !avd_slot_first(pool, f)) + continue; + for (c = 0; c < 3; c++) + if (f->ctx == (void *)&clients[c]) + break; + assert(c < 3 && f->seq > last[c]); + last[c] = f->seq; + f->state = AVD_FRAME_FREE; + completed++; + } + } + for (i = 0; i < 8; i++) { + assert(avd_inbox_full(i, (i + 1) & 7)); + assert(!avd_inbox_full(i, (i + 2) & 7)); + assert(avd_apple_notification(i) == 0x01482cf8U + 0x60 * i); + } + { + struct avd_apple_reply_layout layout = { + .type_word = 0, .type_mask = 31, .client_word = 1, + .client_mask = 1023, .frame_word = 3, .status_word = 4, + .done_type = 8, .verified = true, + }; + struct avd_apple_reply reply; + u32 record[20] = { 7, 9, 0, 1234 }; + + assert(avd_apple_parse_reply(record, 20, &layout, &reply) == -EINPROGRESS); + record[0] = 8; + assert(!avd_apple_parse_reply(record, 20, &layout, &reply)); + assert(reply.frame == 1234 && reply.client == 9 && !reply.status); + record[4] = 1; + assert(!avd_apple_parse_reply(record, 20, &layout, &reply)); + assert(reply.status == -EIO); + assert(avd_apple_parse_reply(record, 19, &layout, &reply) == -EMSGSIZE); + } + assert(completed > 1000); + { + struct avd_frame ordered[AVD_MAX_FRAMES] = {}; + struct avd_ctx *a = (void *)&clients[0], *b = (void *)&clients[1]; + + ordered[1].idx = 1; + ordered[2].idx = 2; + assert(!avd_slot_reserve(&ordered[0], a, 1)); + assert(!avd_slot_reserve(&ordered[1], a, 2)); + assert(!avd_slot_reserve(&ordered[2], b, 3)); + assert(avd_slot_submit_first(ordered, &ordered[0])); + assert(!avd_slot_submit_first(ordered, &ordered[1])); + assert(avd_slot_submit_first(ordered, &ordered[2])); + ordered[0].state = AVD_FRAME_SUBMITTED; + assert(avd_slot_submit_first(ordered, &ordered[1])); + avd_slot_submit_result(&ordered[0], -EBUSY); + assert(!avd_slot_submit_first(ordered, &ordered[1])); + ordered[0].state = AVD_FRAME_DONE; + assert(avd_slot_submit_first(ordered, &ordered[1])); + } + printf("PASS: 100000 shuffled operations, %u ordered retirements\n", completed); + return 0; +} diff --git a/drivers/media/platform/apple/avd/tests/recovery-test.py b/drivers/media/platform/apple/avd/tests/recovery-test.py new file mode 100755 index 0000000000000..66c7e0cdd8453 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/recovery-test.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run production queue recovery and PM transitions against a strict DMA model.""" +import os +from pathlib import Path +import hashlib +import subprocess +import sys +from host_source import function + +HERE = Path(__file__).resolve().parent +ROOT = Path(sys.argv[2]).resolve() if len(sys.argv) > 2 else HERE.parent + +PRELUDE = r''' +#include +#include +#include +#include +#include +#define AVD_MAX_FRAMES 16 +#define AVD_FRAME_FREE 0 +#define AVD_FRAME_SUBMITTED 2 +#define AVD_FRAME_DONE 3 +#define AVD_RECOVER_GAP_MS 20 +#define AVD_STORM_MAX 2 +#define AVD_STORM_WINDOW 60000 +#define msecs_to_jiffies(x) (x) +#define jiffies_to_msecs(x) ((unsigned int)(x)) +#define time_before(a,b) ((long)((a)-(b))<0) +#define time_after(a,b) time_before(b,a) +#define WARN_ONCE(v,...) (assert(!(v)),false) +#define WRITE_ONCE(a,b) ((a)=(b)) +#define spin_lock_irqsave(l,f) do { (void)(l); (f)=0; } while(0) +#define spin_unlock_irqrestore(l,f) do { (void)(l); (void)(f); } while(0) +#define mutex_lock(l) ((void)(l)) +#define mutex_unlock(l) ((void)(l)) +#define dev_warn(d,...) ((void)(d)) +#define dev_err(d,...) ((void)(d)) +#define list_for_each_entry(c,h,n) for((void)(h),(c)=NULL;(c);(c)=NULL) +typedef int irqreturn_t; +struct avd_dev; +struct avd_ctx { bool session_started; }; +struct frame { int state,status; }; +struct avd_backend_ops { + void (*shutdown)(struct avd_dev *); + int (*resume)(struct avd_dev *); + int (*suspend)(struct avd_dev *); +}; +struct queue { + bool (*idle)(struct avd_dev *); + void (*resubmit)(struct avd_dev *); +}; +struct avd_dev { + void *dev,*domain,*empty_domain,*rstc,*compat_active; + const struct avd_backend_ops *backend; + struct { + bool dart_active,dma_parked,irq_enabled,hw_on; + int irq,hw_lock,lock;unsigned int irq_gen; + const struct queue *queue; + } pm; + bool faulted;unsigned int storm_count;unsigned long storm_start,last_recover,rest_until; + struct frame frames[AVD_MAX_FRAMES];int queue_lock,contexts; +}; +static struct avd_dev d; +static unsigned long jiffies=100; +static void *attached; +static int reset_error,boot_error,attach_error; +static unsigned int resets,boots,attaches,detaches,shutdowns,resubmits,drains,irq_opens,irq_closes,step,block_step,reset_step; +static bool pm_runtime_suspended(void *dev) { (void)dev;return false; } +static void disable_irq_nosync(int irq) { assert(irq==1);irq_closes++; } +static void enable_irq(int irq) { assert(irq==1);assert(!d.pm.dma_parked);irq_opens++; } +static int iommu_attach_device(void *domain,void *dev) { + (void)dev;assert(domain==d.empty_domain&&!d.pm.hw_on&&!d.pm.irq_enabled); + attaches++;if(attach_error)return attach_error;attached=domain;block_step=++step;return 0; +} +static void iommu_detach_device(void *domain,void *dev) { + (void)dev;assert(domain==d.empty_domain&&attached==domain&&!reset_error); + detaches++;attached=d.domain; +} +static void *iommu_get_domain_for_dev(void *dev) { (void)dev;return attached; } +static int reset_control_reset(void *rstc) { + (void)rstc;assert(!d.domain||attached==d.empty_domain);assert(!d.pm.hw_on&&!d.pm.irq_enabled); + resets++;reset_step=++step;assert(!d.domain||block_stepframes[i].state!=AVD_FRAME_FREE)return false; + return true; +} +static void resubmit(struct avd_dev *avd) { (void)avd;resubmits++; } +static void avd_pm_drain_unmaps(struct avd_dev *avd) { assert(avd->pm.dart_active);drains++; } +''' + +MAIN = r''' +static const struct avd_backend_ops backend={.shutdown=shutdown_backend}; +static const struct queue queue={.idle=idle,.resubmit=resubmit}; +static int domain_token,empty_token; +static void init(void) { + d=(struct avd_dev){.dev=&d,.domain=&domain_token,.empty_domain=&empty_token,.backend=&backend, + .pm={.dart_active=true,.irq_enabled=true,.hw_on=true,.irq=1,.queue=&queue}}; + attached=d.domain;reset_error=boot_error=attach_error=0; + resets=boots=attaches=detaches=shutdowns=resubmits=drains=irq_opens=irq_closes=step=0; +} +int main(void) { + init();assert(!avd_pm_recover(&d));assert(resets==1&&boots==1&&attaches==1&&detaches==1); + assert(!d.pm.dma_parked&&!d.pm.irq_enabled&&attached==d.domain); + avd_pm_enable_irq(&d);assert(irq_opens==1); + init();reset_error=-EIO;d.frames[0].state=AVD_FRAME_SUBMITTED;avd_reset_queue(&d,-ETIMEDOUT); + assert(d.faulted&&resets==1&&!boots&&!detaches&&d.pm.dma_parked&&attached==d.empty_domain); + assert(d.frames[0].state==AVD_FRAME_DONE&&!d.pm.irq_enabled); + d.frames[0].state=AVD_FRAME_FREE; + assert(!avd_pm_power_off(&d)&&!avd_pm_power_on(&d)); + assert(!boots&&!resubmits&&!d.pm.irq_enabled&&attached==d.empty_domain); + init();boot_error=-ETIMEDOUT;avd_reset_queue(&d,-ETIMEDOUT); + assert(d.faulted&&resets==1&&boots==1&&attaches==2&&detaches==1&&d.pm.dma_parked); + assert(attached==d.empty_domain&&!d.pm.irq_enabled); + init();avd_reset_queue(&d,-EIO);avd_reset_queue(&d,-EIO);avd_reset_queue(&d,-EIO); + assert(resets==2&&boots==2&&attaches==3&&detaches==2&&!d.faulted&&d.rest_until); + assert(d.pm.dma_parked&&attached==d.empty_domain&&!d.pm.irq_enabled); + unsigned int old_boots=boots,old_resets=resets; + assert(!avd_pm_power_off(&d)&&!avd_pm_power_on(&d)); + assert(boots==old_boots&&resets==old_resets&&!resubmits&&!d.pm.irq_enabled); + /* The expiry path (dispatch-test) calls recover under a PM reference. */ + jiffies=d.rest_until;assert(!avd_pm_recover(&d));assert(resets==3&&!d.pm.dma_parked); + init();attach_error=-EIO;assert(avd_pm_recover(&d)==-EIO);assert(!resets&&!boots&&!detaches); + init();d.pm.dart_active=false;assert(avd_pm_park(&d)==-EHOSTDOWN);assert(!shutdowns&&!resets&&!attaches); + puts("PASS: production recovery/PM: park before reset, failed reset/boot stays blocked, budget rests, parked resume never boots, expiry recovery"); +} +''' + + +def main(): + out = Path(sys.argv[1]).resolve() + out.mkdir(parents=True, exist_ok=True) + pm = (ROOT / 'avd-pm.c').read_text() + queue = (ROOT / 'avd-queue.c').read_text() + (out / 'inputs.sha256').write_text(''.join( + hashlib.sha256((ROOT / n).read_bytes()).hexdigest() + ' ' + n + '\n' + for n in ['avd-pm.c', 'avd-queue.c'])) + code = out / 'recovery.c' + code.write_text(PRELUDE + ''.join(function(pm, n) for n in [ + 'avd_pm_enable_irq', 'avd_pm_disable_irq', 'avd_pm_park', 'avd_pm_recover', + 'avd_pm_power_on', 'avd_pm_power_off']) + + function(queue, 'avd_reset_queue') + MAIN) + exe = out / 'recovery-test' + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', + '-Werror', '-fsanitize=address,undefined', '-g', str(code), + '-o', str(exe)], check=True) + subprocess.run([str(exe)], check=True) + + +if __name__ == '__main__': + main() diff --git a/drivers/media/platform/apple/avd/tests/run-host.sh b/drivers/media/platform/apple/avd/tests/run-host.sh new file mode 100755 index 0000000000000..70e287f14732d --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/run-host.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only OR MIT +set -eu +avd_test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +avd_test_out=${1:?usage: run-host.sh OUTPUT_BINARY} +${CC:-cc} -std=gnu11 -Wall -Wextra -Werror -O2 \ + -I"$avd_test_dir/include" "$avd_test_dir/queue-host.c" -o "$avd_test_out" +"$avd_test_out" diff --git a/drivers/media/platform/apple/avd/tests/unicorn-inflight-test.py b/drivers/media/platform/apple/avd/tests/unicorn-inflight-test.py new file mode 100755 index 0000000000000..1ba0f5d0f121f --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/unicorn-inflight-test.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Routed CM3 queue test using production host programs, DMA sites and MMIO. + +usage: unicorn-inflight-test.py FIRMWARE_TREE IMAGE PROGRAM_DIRECTORY [--before] +The decoder model holds each VP busy until an explicit DONE, checks emitted +addresses, and delays DONE/PP/outbox to exercise overlap. It does not decode +pixels or simulate the decoder's undocumented internal register execution. +""" +import importlib.util +from pathlib import Path +import struct +import sys +from unicorn.arm_const import UC_ARM_REG_SP + +firmware, image, programs = map(lambda p: Path(p).resolve(), sys.argv[1:4]) +spec = importlib.util.spec_from_file_location('transport', firmware / 'tests/transport-test.py') +transport = importlib.util.module_from_spec(spec) +spec.loader.exec_module(transport) + + +class LiveVPError(AssertionError): + pass + + +class Inflight(transport.Transport): + def __init__(self): + self.booting = False + self.live = None + self.feed_idx = 0 + self.feed_word = 0 + self.expected = [] + self.events = [] + self.address_checks = 0 + self.pp_owner = None + self.vp_events = self.pp_events = 0 + super().__init__(image) + self.booting = True + self.uc.reg_write(UC_ARM_REG_SP, self.u32(0)) + self.uc.emu_start(self.u32(4), 0, count=500000) + assert not self.booting, 'reset did not publish boot-ready' + assert self.u32(0x50010010) & 2 + self.lo = 0x150 if self.version == 3 else 0x1d0 + self.hi = None if self.version == 3 else 0x20c + self.mask = 0x18c if self.version == 3 else 0x248 + self.cach = 0x1c8 if self.version == 3 else 0x284 + self.xfer = 0x204 if self.version == 3 else 0x2c0 + + def route(self, irq): + if irq < 14: + router = self.u32(0x50010010) & (1 << irq) + else: + off = irq - 14 + router = self.u32(0x50010014 + (off // 32) * 4) & (1 << (off % 32)) + nvic = self.u32(0xe000e100 + (irq // 32) * 4) & (1 << (irq % 32)) + assert router and nvic, f'masked IRQ{irq}' + self.call(f'irq{irq}') + + def fifo_addr(self, idx): + low = self.u32(self.decode + self.lo + idx * 4) + addr = low << 8 if self.hi is None else low | self.u32(self.decode + self.hi + idx * 4) << 32 + assert addr, f'zero FIFO DMA address at slot {idx}' + self.address_checks += 1 + return addr + + def write(self, uc, access, addr, size, value, user): + if self.booting and addr == 0x50010090 and value == 1: + self.booting = False + uc.emu_stop() + if self.decode + self.fifo <= addr < self.decode + self.fifo + self.vps * 4: + idx, words = self.expected[self.feed_idx] + assert addr == self.decode + self.fifo + 4 * 4, 'wrong H.264 VP' + assert value == words[self.feed_word], (idx, self.feed_word, hex(value)) + if self.feed_word == 0: + if self.live is not None: + raise LiveVPError(f'VP4 receives slot {idx} START while slot {self.live} has no DONE') + self.live = idx + assert (value >> 4) & 15 == idx + fifo = self.fifo_addr(idx) + self.events.append(f'VP4 START slot={idx} fifo={fifo:#x}') + self.feed_word += 1 + if self.feed_word == len(words): + self.feed_idx += 1 + self.feed_word = 0 + if addr == self.decode + self.pp: + idx = (value >> 4) & 15 + assert self.pp_owner is None, 'PP overwritten before DONE' + assert value == 0x2b000200 | idx << 4 | self.info[10] + self.pp_owner = idx + fifo = self.fifo_addr(idx) + self.events.append(f'PP START slot={idx} fifo={fifo:#x}') + if addr == 0x50010060: + self.events.append(f'completion token={value:#x}') + super().write(uc, access, addr, size, value, user) + + def submit_host(self, idx, start, tag): + variant = image.stem.removeprefix('avd-fw-') + source = programs / variant + data = (source / f'descriptor-{idx}.bin').read_bytes() + desc = list(struct.unpack('<16I', data)) + words = list(struct.unpack('<' + 'I' * desc[6], (source / f'h264-{idx}.bin').read_bytes())) + assert desc[:4] == [0x41565131, 0x10 | idx, 1, 4] + assert desc[4:6] == [0, 0] and desc[8:12] == [1, 0, 0, 1] + assert desc[12:] == [0, 0, 0, 0] + assert desc[7] == 0x2b000200 | idx << 4 | self.info[10] + # program_lo is a free-running SRAM word count, not a DMA pointer. + desc[4] = start + desc[1] = tag + assert words[0] == 0x2b000200 | idx << 4 + assert words[-1] == 0x2b000400 | idx << 4 + for line in (source / f'dma-sites-{idx}.txt').read_text().splitlines(): + offset, width, shift, expected = line.split() + offset, width, shift, expected = int(offset), int(width), int(shift), int(expected, 16) + encoded = sum(words[offset + n] << (32 * n) for n in range(width)) + assert encoded and encoded == expected >> shift, f'zero/truncated DMA operand slot {idx} word {offset}' + self.address_checks += 1 + for n, word in enumerate(words[:-1]): + if word >> 20 == 0x2d8: # coded-slice address: high 13 bits + next low word + assert ((word & 0x1fff) << 32) | words[n + 1], f'zero coded-data DMA slot {idx}' + self.address_checks += 1 + snapshot = (source / f'mmio-{idx}.bin').read_bytes() + # Replay only this host submission's writes, preserving live slots. + offsets = [self.lo, self.mask, self.cach, self.xfer] + if self.hi is not None: + offsets.append(self.hi) + for off in offsets: + at = off + idx * 4 + self.put(self.decode + at, struct.unpack_from(' retired: + idx = retired % slots + assert t.result(idx) == [2, 0, tags[retired]] + assert t.notifications[retired] == 0x54530000 | tags[retired] + retired += 1 + t.busy = False + assert t.vp_events == t.pp_events == depth + assert t.feed_idx == depth and not t.feed_word and t.live is t.pp_owner is None + assert t.u32(0x10007ff8) == t.u32(0x10007ffc) == wp + assert not t.fetches and not t.ring_writes + if depth in (2, 15): + print('\n'.join('TRACE ' + event for event in t.events), flush=True) + print(f'PASS {image.stem}: workload {depth}, peak {peak}, {t.address_checks} nonzero DMA checks, ' + f'{depth} routed VP/PP completions, RP=WP={wp}', flush=True) + + +if '--before' in sys.argv[4:]: + try: + check(2) + except LiveVPError as error: + print(f'REPRODUCED {image.stem}: {error}', flush=True) + else: + raise AssertionError('baseline unexpectedly respected VP ownership') +else: + for depth in range(2, 16): + check(depth) diff --git a/drivers/media/platform/apple/avd/tests/unicorn-queue-test.py b/drivers/media/platform/apple/avd/tests/unicorn-queue-test.py new file mode 100755 index 0000000000000..d1b08ce00f55e --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/unicorn-queue-test.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run codec-generated Linux descriptors/programs through actual CM3 code. + +Usage: unicorn-queue-test.py FIRMWARE_TREE IMAGE PROGRAM_DIRECTORY [--broken] +The firmware tree provides its existing tests/transport-test.py MMIO harness. +Extend that harness with reset-vector boot, real FIFO status bits, IRQ router +and NVIC checks, and VP/PP events following a complete generated program. +Decoder arithmetic/DMA/pixels remain synthetic; this is a transport regression. +--broken proves the old image stalls both at IRQ routing and at publication. +""" +import importlib.util +from pathlib import Path +import struct +import sys + +from unicorn.arm_const import UC_ARM_REG_SP + +firmware = Path(sys.argv[1]).resolve() +image = Path(sys.argv[2]).resolve() +programs = Path(sys.argv[3]).resolve() +if (programs / 'v3-t1').is_dir(): + programs /= 'v3-t1' +broken = '--broken' in sys.argv[4:] +spec = importlib.util.spec_from_file_location('transport', firmware / 'tests/transport-test.py') +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class RoutedTransport(module.Transport): + def __init__(self, image): + self.booting = False + self.suppressed = set() + super().__init__(image) + self.booting = True + self.uc.reg_write(UC_ARM_REG_SP, self.u32(0)) + self.uc.emu_start(self.u32(4), 0, count=500000) + assert not self.booting, 'reset vector did not publish boot-ready' + assert self.u32(0x50010010) & 2, 'CM3 mailbox-0 not-empty disabled' + # Linux avd_boot() AP mailbox-1 notification routing, separate from CM3. + self.ap_irq_enable = 8 + + def read(self, uc, access, addr, size, value, user): + super().read(uc, access, addr, size, value, user) + if addr == 0x5001005c: + # BIT(2) belongs to IRQ status, not this FIFO status register. + # Empty FIFO has bit 17 set, ENABLE remains bit 0. + self.put(addr, 1 | (0x10100 if self.busy else 1 << 17)) + + def write(self, uc, access, addr, size, value, user): + super().write(uc, access, addr, size, value, user) + if self.booting and addr == 0x50010090 and value == 1: + self.booting = False + uc.emu_stop() + if addr == 0x50010060: + assert self.ap_irq_enable == 8, 'AP mailbox-1 IRQ disabled' + + def route(self, irq): + if irq < 14: + router = self.u32(0x50010010) & (1 << irq) + else: + offset = irq - 14 + router = self.u32(0x50010014 + (offset // 32) * 4) & (1 << (offset % 32)) + nvic = self.u32(0xe000e100 + (irq // 32) * 4) & (1 << (irq % 32)) + if not (router and nvic): + self.suppressed.add(irq) + return False + self.call('irq' + str(irq)) + return True + + def descriptor(self, idx): + data = (programs / f'descriptor-{idx}.bin').read_bytes() + d = struct.unpack('<16I', data) + assert d[0:4] == (0x41565131, 0x10 | idx, 1, 4) + assert d[4:6] == (0, 0) and d[8:12] == (1, 0, 0, 1) + assert d[7] == 0x2b00020f | (idx << 4) + assert d[12:] == (0, 0, 0, 0) + self.uc.mem_write(0x10008000 + idx * 64, data) + # Actual Linux t8112_configure_stream() results are asserted by program-test. + self.put(self.decode + 0x10c, 7) # VP4 decoder-local mask + self.put(self.decode + 0x120, 5) # PP decoder-local mask + self.put(0x50010058, self.ring + idx * 64) + assert self.route(1), 'AP mailbox-0 doorbell was not delivered' + words = list(struct.unpack('<' + 'I' * d[6], (programs / f'h264-{idx}.bin').read_bytes())) + assert words[0] == 0x2b000200 | (idx << 4) + assert words[-1] == 0x2b000400 | (idx << 4) + module.ring_write(self, 0, words) + self.put(0x10007ffc, len(words)) + return words + + def execute(self, words): + vp_done = pp_done = False + for _ in range(40): + self.poll() + if not vp_done and self.stores == words: + # Generate DONE only after the complete real codec program was fed. + assert self.u32(self.decode + 0x10c) & 4 + vp_done = self.route(100) # v3 H.264 VP4 DONE + if self.kicks and not pp_done: + assert self.u32(self.decode + 0x120) & 4 + pp_done = self.route(64) + assert self.stores == words + assert self.u32(0x10007ff8) == self.u32(0x10007ffc) == len(words) + assert not self.fetches and not self.ring_writes + return vp_done, pp_done + + +def force_decoder_routes(t): + # Isolate the independent status-register bug in the unfixed image. + for irq in (100, 64): + offset = irq - 14 + addr = 0x50010014 + (offset // 32) * 4 + t.put(addr, t.u32(addr) | (1 << (offset % 32))) + + +assert image.stem == 'avd-fw-v3-t1', 'this first-light regression targets T8112' +if broken: + t = RoutedTransport(image) + words = t.descriptor(0) + assert t.execute(words) == (False, False) + assert 100 in t.suppressed and t.result(0) == [1, 0, 0] and not t.notifications + print('REPRODUCED: v3 VP4 IRQ100 masked; program consumed but descriptor remains SUBMITTED') + t = RoutedTransport(image) + words = t.descriptor(0) + force_decoder_routes(t) + assert t.execute(words) == (True, True) + assert t.kicks == [0x2b00020f] + assert t.result(0) == [1, 0, 0] and not t.notifications + print('REPRODUCED: VP/PP done with routes repaired; EMPTY bit17 never passes firmware BIT(2) test') +else: + for idx in (0, 1, 14): + t = RoutedTransport(image) + words = t.descriptor(idx) + assert t.execute(words) == (True, True) + assert t.u32(0x50010018) == 0x50000 # PP unknown/DONE + assert t.u32(0x5001001c) == 0x700000 # VP4 unknown/error/DONE only + assert all(t.u32(a) == 0 for a in (0x50010014, 0x50010020, 0x50010024, 0x50010028)) + assert t.kicks == [0x2b00020f | (idx << 4)] + assert t.result(idx) == [2, 0, 0x10 | idx] + assert t.notifications == [0x54530010 | idx] + print(f'PASS: v3-t1 slot {idx}, {len(words)} production H.264 words, ' + f'IRQ1 -> VP4 IRQ100 -> PP IRQ64 -> tagged completion {t.notifications[0]:#x}') diff --git a/drivers/media/platform/apple/avd/tests/unicorn-vp9-test.py b/drivers/media/platform/apple/avd/tests/unicorn-vp9-test.py new file mode 100644 index 0000000000000..72234b3d5b1a9 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/unicorn-vp9-test.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""VP9 production programs: VP(N+1) starts after VP(N) DONE, during PP(N). + +Usage: unicorn-vp9-test.py FIRMWARE_TREE v3-t1_IMAGE VP9_HOST_OUTPUT +The symbol counts are a host fixture. CM3 code, descriptors, program words, +PP commands and configure_stream writes are production. No pixels are decoded. +""" +import importlib.util +from pathlib import Path +import struct +import sys +from unicorn.arm_const import UC_ARM_REG_SP + +fw, image, programs = map(lambda p: Path(p).resolve(), sys.argv[1:4]) +spec = importlib.util.spec_from_file_location('transport', fw / 'tests/transport-test.py') +transport = importlib.util.module_from_spec(spec) +spec.loader.exec_module(transport) + + +class VP9(transport.Transport): + def __init__(self): + self.booting = False + self.expected = [] + self.events = [] + self.live = self.pp_owner = None + self.feed = self.word = 0 + super().__init__(image) + assert self.version == 3 and self.tier == 1 + self.booting = True + self.uc.reg_write(UC_ARM_REG_SP, self.u32(0)) + self.uc.emu_start(self.u32(4), 0, count=500000) + assert not self.booting + + def route(self, irq): + if irq < 14: + router = self.u32(0x50010010) & (1 << irq) + else: + off = irq - 14 + router = self.u32(0x50010014 + off // 32 * 4) & (1 << (off % 32)) + nvic = self.u32(0xe000e100 + irq // 32 * 4) & (1 << (irq % 32)) + assert router and nvic, f'IRQ {irq} masked' + self.call(f'irq{irq}') + + def write(self, uc, access, addr, size, value, user): + if self.booting and addr == 0x50010090 and value == 1: + self.booting = False + uc.emu_stop() + if addr == self.decode + self.fifo + 8 * 4: + idx, words = self.expected[self.feed] + assert value == words[self.word], (idx, self.word, value) + if not self.word: + assert self.live is None, 'successor issued before VP DONE' + self.live = idx + self.events.append(f'VP9 VP START {idx}; PP owner={self.pp_owner}') + self.word += 1 + if self.word == len(words): + self.feed += 1 + self.word = 0 + if addr == self.decode + self.pp: + idx = (value >> 4) & 15 + assert self.pp_owner is None, 'successor PP issued before predecessor PP DONE' + assert value == 0x2b00020f | idx << 4 + self.pp_owner = idx + self.events.append(f'PP START {idx}') + if addr == 0x50010060: + self.events.append(f'notification {value:#x}') + super().write(uc, access, addr, size, value, user) + + def submit_host(self, idx, start): + stem = f'vp9-adaptive-{idx}' + data = (programs / f'{stem}.bin').read_bytes() + words = list(struct.unpack('<' + 'I' * (len(data) // 4), data)) + desc = list(struct.unpack('<16I', (programs / f'{stem}-desc.bin').read_bytes())) + assert desc[:4] == [0x41565131, 0x10 | idx, 2, 8] + assert desc[4:6] == [0, 0] and desc[6] == len(words) + assert desc[8:16] == [1, 0, 0, 3, 0, 0, 0, 0] + assert words[0] == 0x2b000200 | idx << 4 + assert words[-1] == 0x2b000400 | idx << 4 + mmio = (programs / f'{stem}-mmio.bin').read_bytes() + for offset in (0x150, 0x18c, 0x1c8, 0x204): + at = offset + idx * 4 + self.put(self.decode + at, struct.unpack_from(' 2 else HERE.parent +KERNEL = Path(sys.argv[3]).resolve() if len(sys.argv) > 3 else AVD.parents[4] +OUT = Path(sys.argv[1]).resolve() +OUT.mkdir(parents=True, exist_ok=True) + +# Share only the explicit MMIO/allocation boundary with the H.264 tests. +spec = importlib.util.spec_from_file_location('program_fixture', HERE / 'program-test.py') +fixture = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fixture) +pre = fixture.PRELUDE +pre = pre.replace('typedef uint8_t u8;', 'typedef uint8_t u8; typedef uint16_t u16;') +pre = pre.replace('#define AVD_MAX_INST 512', '#define AVD_MAX_INST 512\n#define AVD_CODEC_HEVC 0') +pre = pre.replace('struct vb2_buffer { struct { size_t length; } planes[1]; dma_addr_t dma; };', + 'struct vb2_buffer { struct { size_t length; } planes[1]; dma_addr_t dma; void *cpu; size_t payload; u64 timestamp; };') +pre = pre.replace('struct avd_comp comp; unsigned int bank;', 'struct avd_comp comp; unsigned int bank; struct { unsigned int width,height,bit_depth; } vp9;') +pre = pre.replace('struct vb2_v4l2_buffer *dst; } bufs;', 'struct vb2_v4l2_buffer *src,*dst; } bufs;') +pre = pre.replace('unsigned int fifo_idx, pp_total;', 'unsigned int fifo_idx, pp_total, max_frames; int ctrl_hdl;') +pre = pre.replace('static int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n)', + '#ifdef AVD_TEST_VP_READY\nstatic struct avd_ctx *allocating_ctx;\nstatic int fail_alloc_after;\nstatic bool avd_vp9_ready(struct avd_ctx *);\n#endif\nstatic int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n)') +pre = pre.replace('{ (void)d; free(b->cpu); b->cpu = calloc(1,n);', + '{\n#ifdef AVD_TEST_VP_READY\n if(allocating_ctx){assert(!avd_vp9_ready(allocating_ctx));if(fail_alloc_after&&!--fail_alloc_after)return -ENOMEM;}\n#endif\n (void)d; free(b->cpu); b->cpu = calloc(1,n);') +pre += r''' +#define round_up(x,a) (((x)+(a)-1)&~((a)-1)) +#define max_t(t,a,b) ((t)(a)>(t)(b)?(t)(a):(t)(b)) +#define min(a,b) ((a)<(b)?(a):(b)) +#define clamp(x,l,h) ((x)<(l)?(l):(x)>(h)?(h):(x)) +#define WARN_ON(v) (assert(!(v)),false) +#define GFP_KERNEL 0 +#define smp_load_acquire(p) (*(p)) +#define smp_store_release(p,v) (*(p)=(v)) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define noinline_for_stack __attribute__((noinline)) +#define static_assert _Static_assert +#define EXPORT_SYMBOL_GPL(x) +#define MODULE_LICENSE(x) +#define MODULE_DESCRIPTION(x) +#define MODULE_AUTHOR(x) +#define VB2_BUF_STATE_DONE 0 +#define VB2_BUF_STATE_ERROR 1 +enum vb2_buffer_state { STATE_DUMMY }; +#define dev_err(d,...) ((void)(d)) +#define kzalloc(n,f) calloc(1,n) +#define kfree free +static bool vp_feature = true; +static bool avd_qfw_has_vp_done(const struct avd_dev *d) { return d->queued && vp_feature; } +static void *vb2_plane_vaddr(struct vb2_buffer *b,int p) { (void)p;return b->cpu; } +static size_t vb2_get_plane_payload(struct vb2_buffer *b,int p) { (void)p;return b->payload; } +static struct vb2_v4l2_buffer *current_src,*current_dst; +static struct vb2_v4l2_buffer *v4l2_m2m_next_src_buf(void *m) { (void)m;return current_src; } +static u32 get_unaligned_be32(const void *p) { const u8 *v=p;return (u32)v[0]<<24|(u32)v[1]<<16|(u32)v[2]<<8|v[3]; } +struct v4l2_ctrl { union { void *p; } p_cur; }; +static struct v4l2_ctrl_vp9_frame params; +static struct v4l2_ctrl_vp9_compressed_hdr updates; +static struct v4l2_ctrl frame_ctrl,update_ctrl; +static struct v4l2_ctrl *v4l2_ctrl_find(void *h,u32 id) { + (void)h;frame_ctrl.p_cur.p=¶ms;update_ctrl.p_cur.p=&updates; + return id==V4L2_CID_STATELESS_VP9_FRAME?&frame_ctrl:&update_ctrl; +} +static void avd_run_preamble(struct avd_ctx *ctx,struct avd_run *run) { + (void)ctx;memset(run,0,sizeof(*run));run->bufs.src=current_src;run->bufs.dst=current_dst; + run->coded_in=current_src->vb2_buf.dma;run->comp_out=current_dst->vb2_buf.dma; + run->y_out=run->comp_out+0x100000;run->uv_out=run->comp_out+0x300000; +} +static void avd_run_postamble(struct avd_ctx *ctx,struct avd_run *run) { (void)ctx;(void)run; } +static int avd_init_job(struct avd_ctx *ctx,int codec,size_t segments) { + free(ctx->job.segments);ctx->job.codec=codec;ctx->job.num=0; + ctx->job.segments=calloc(segments,sizeof(*ctx->job.segments));return ctx->job.segments?0:-ENOMEM; +} +static int avd_submit_job(struct avd_ctx *ctx); +''' +pre = pre.replace('dma_addr_t comp_out, y_out, uv_out;', 'dma_addr_t comp_out, y_out, uv_out, coded_in;') + + +def without_includes(s): + return re.sub(r'^#include[^\n]*\n', '', s, flags=re.M) + +vp9 = (AVD / 'avd-vp9.c').read_text() +helper_h = (KERNEL / 'include/media/v4l2-vp9.h').read_text() +helper_c = (KERNEL / 'drivers/media/v4l2-core/v4l2-vp9.c').read_text() +source = '#define AVD_TEST_VP_READY\n' + pre + '\n' + without_includes(helper_h) + '\n' + without_includes(helper_c) +source += '\n' + without_includes((AVD / 'avd-inst.h').read_text()) +# Include unchanged codec declarations/functions, excluding only the ops table. +source += '\n' + without_includes(vp9[vp9.index('#define VP9_Q_IDX'):vp9.index('static enum avd_image_fmt avd_vp9_get_image_fmt')]) +source += '\n' + function(vp9, 'avd_vp9_pp_kicks') +source += '\n' + function((AVD / 'avd-backend-open.c').read_text(), 'avd_open_build_program') +qfw = (AVD / 'avd-backend-queue.c').read_text() +source += qfw[qfw.index('#define AVDQ_MAGIC'):qfw.index('/* Arguments of dev_dbg()')] +hw = (AVD / 'avd-hw.c').read_text() +source += hw[hw.index('#define AVD_V3_'):hw.index('int avd_boot(')] +source += '#define w32(r,v) writel((v),avd->ctrl+(r))\n#define m32(r,v) w32(r,(v)|readl(avd->ctrl+(r)))\n' +source += function(hw, 't8112_configure_stream') +source += function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') +source += '\nstatic int avd_submit_job(struct avd_ctx *ctx) { return avd_open_build_program(ctx); }\n' +source += r''' +static void write_program(const char *dir,const char *name,struct avd_frame *f) { + char path[4096];snprintf(path,sizeof(path),"%s/%s.bin",dir,name);FILE *out=fopen(path,"wb");assert(out); + assert(fwrite(f->prog->cpu,4,f->prog_words,out)==f->prog_words);fclose(out); + struct avd_dev *dev=f->ctx->dev;struct avd_qfw *q=dev->qfw; + /* The descriptor, PP command and configure_stream snapshot are production. */ + q->wp=0;memset(q->sram,0,0x10000);f->tag=0x10|f->idx; + assert(!avd_qfw_submit(dev,f)); + snprintf(path,sizeof(path),"%s/%s-desc.bin",dir,name);out=fopen(path,"wb");assert(out); + assert(fwrite(q->sram+0x8000+f->idx*64,1,64,out)==64);fclose(out); + snprintf(path,sizeof(path),"%s/%s-mmio.bin",dir,name);out=fopen(path,"wb");assert(out); + assert(fwrite(dev->ctrl,1,0x10000,out)==0x10000);fclose(out); +} +static unsigned int offset_words[256]; +static void save_dma(const char *dir,const char *name) { + char path[4096];snprintf(path,sizeof(path),"%s/%s-dma.txt",dir,name);FILE *out=fopen(path,"w");assert(out); + for(unsigned int i=0;inbanks==2); + assert(p->bank[0].bufs.counts.addr!=p->bank[1].bufs.counts.addr); + assert(p->bank[0].bufs.probs.addr!=p->bank[1].bufs.probs.addr); + assert(p->bank[0].cnts.skip!=p->bank[1].cnts.skip); + assert(p->bank[0].cnts.coeff[0][0][0][0][0]!=p->bank[1].cnts.coeff[0][0][0][0][0]); + assert(p->bank[0].cnts.eob[0][0][0][0][0][0]!=p->bank[1].cnts.eob[0][0][0][0][0][0]); + /* All header-flag combinations, including malformed error-resilient refresh. */ + for(unsigned int bits=0;bits<8;bits++) { + u32 flags=(bits&1?V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX:0)| + (bits&2?V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE:0)|(bits&4?V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT:0); + assert(avd_vp9_needs_counts(flags)==(bits==1)); + } + params=(struct v4l2_ctrl_vp9_frame){.flags=V4L2_VP9_FRAME_FLAG_KEY_FRAME|V4L2_VP9_FRAME_FLAG_SHOW_FRAME|V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX, + .frame_width_minus_1=1919,.frame_height_minus_1=1087,.bit_depth=8}; + updates.tx_mode=V4L2_VP9_TX_MODE_SELECT; + current_dst=&dst[0].base.vb;frame.idx=0;dma_count=0; + assert(avd_vp9_ready(&ctx));assert(!avd_vp9_run(&ctx)); + assert(dst[0].bank==1&&p->bank[0].busy&&!avd_vp9_ready(&ctx)); + assert(frame.fifo_iova==p->bank[0].bufs.inst.addr); + write_program(argv[1],"vp9-adaptive-0",&frame); + /* Symbol-count fixture must change a saved coefficient probability. */ + struct avd_vp9_frame_symbol_counts *counts=p->bank[0].bufs.counts.cpu; + for(size_t i=0;i<528;i++){counts->eob_0[i]=16;counts->ref_cnt[i*4]=1;counts->ref_cnt[i*4+1]=4;counts->ref_cnt[i*4+2]=11;counts->ref_cnt[i*4+3]=8;} + struct v4l2_vp9_frame_context before=p->frame_context[0]; + avd_vp9_vp_done(&ctx,&dst[0].base.vb); + assert(avd_vp9_ready(&ctx)&&p->bank[0].busy&&p->bank[0].updated&&dst[0].bank==1); + assert(memcmp(&before,&p->frame_context[0],sizeof(before))); + struct v4l2_vp9_frame_context adapted=p->frame_context[0]; + avd_vp9_vp_done(&ctx,&dst[0].base.vb);assert(!memcmp(&adapted,&p->frame_context[0],sizeof(adapted))); + current_dst=&dst[1].base.vb;frame.idx=1;dma_count=0;params.flags=V4L2_VP9_FRAME_FLAG_SHOW_FRAME|V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX; + reference=dst[0];reference.vp9.width=1920;reference.vp9.height=1088; + assert(!avd_vp9_run(&ctx));assert(dst[1].bank==2&&p->bank[0].busy&&p->bank[1].busy); + assert(!memcmp(&adapted,&p->bank[1].probability_tables,sizeof(adapted))); + write_program(argv[1],"vp9-adaptive-1",&frame); + assert(!avd_vp9_ready(&ctx)); + struct avd_vp9_run attempt; + current_dst=&dst[2].base.vb;assert(avd_vp9_run_preamble(&ctx,&attempt)==-EBUSY);assert(!dst[2].bank); + /* PP(0) retires while b points to bank 1: only bank 0 may be released. */ + avd_vp9_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE); + assert(!p->bank[0].busy&&p->bank[1].busy&&!dst[0].bank); + assert(!memcmp(&adapted,&p->frame_context[0],sizeof(adapted))); + /* Failed VP/prepare must never adapt partial counts. */ + avd_vp9_done(&ctx,&src,&dst[1].base.vb,VB2_BUF_STATE_ERROR); + assert(!p->bank[1].busy&&avd_vp9_ready(&ctx)); + assert(!memcmp(&adapted,&p->frame_context[0],sizeof(adapted))); + for(unsigned int mode=0;mode<3;mode++) { + params.flags=mode==0?0:mode==1?V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX|V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE:V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT; + current_dst=&dst[0].base.vb;frame.idx=0;assert(!avd_vp9_run(&ctx)); + assert(avd_vp9_ready(&ctx)&&p->bank[0].busy); + if(mode==0) { + /* The third inter picture's temporal-MV input stays in the prior bank. */ + assert(p->last.color==p->bank[0].bufs.color.addr); + bool prior_color=false;u32 *words=program.cpu; + for(size_t j=0;jbank[1].bufs.color.addr>>8))prior_color=true; + assert(prior_color); + } + current_dst=&dst[1].base.vb;frame.idx=1;assert(!avd_vp9_run(&ctx)); + assert(p->bank[0].busy&&p->bank[1].busy); + avd_vp9_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE);avd_vp9_done(&ctx,&src,&dst[1].base.vb,VB2_BUF_STATE_DONE); + } + avd_vp9_stop(&ctx);assert(!avd_vp9_ready(&ctx)); + /* Old firmware and the legacy transport allocate only one bank. */ + for(unsigned int mode=0;mode<2;mode++) { + dev.queued=mode;vp_feature=false;assert(!avd_vp9_start(&ctx));p=ctx.priv; + assert(p->nbanks==1&&ctx.max_frames==1);avd_vp9_stop(&ctx); + } + dev.queued=true;vp_feature=true;fail_alloc_after=13;allocating_ctx=&ctx; + assert(avd_vp9_start(&ctx)==-ENOMEM);allocating_ctx=NULL; + assert(!ctx.priv&&!avd_vp9_ready(&ctx)&&ctx.max_frames==1); + free(program.cpu);free(queue.sram);free(dev.ctrl);free(dev.mbox);puts("PASS: VP9 dependency table, bank ownership, real adaptation, early release and legacy fallback"); +} +''' +# Extra pixel-format shape and CPU-reference dimensions in this boundary. +source = source.replace('struct { u32 bytesperline; } plane_fmt[1];', 'u32 width,height; struct { u32 bytesperline; } plane_fmt[1];') +(OUT/'vp9-host.c').write_text(source) +cmd = [os.environ.get('CC','cc'), '-std=gnu11','-g','-O1','-Wall','-Wextra','-Werror', + '-Wno-unused-function','-Wno-unused-parameter','-Wno-sign-compare', + '-fsanitize=address,undefined','-fno-omit-frame-pointer',str(OUT/'vp9-host.c'),'-o',str(OUT/'vp9-host')] +subprocess.run(cmd,check=True) +subprocess.run([str(OUT/'vp9-host'),str(OUT)],check=True) +(OUT/'source.sha256').write_text('\n'.join(hashlib.sha256(p.read_bytes()).hexdigest()+' '+str(p) for p in (AVD/'avd-vp9.c',AVD/'avd-backend-open.c',KERNEL/'include/media/v4l2-vp9.h',KERNEL/'drivers/media/v4l2-core/v4l2-vp9.c'))+'\n') + +# Optional independent before/after executable comparison. The baseline is read +# from public Git history, never from a captured stock firmware program. +base_ref = os.environ.get('AVD_VP9_BASELINE_REF') +if base_ref: + old = subprocess.check_output(['git','-C',str(KERNEL),'show', + base_ref + ':drivers/media/platform/apple/avd/avd-vp9.c'],text=True) + legacy_main = r''' +int main(int argc,char **argv) { + assert(argc==2); + struct avd_variant variant={.revision=4,.quirks=AVD_QUIRK_LSR,.fifo_slots=15}; + struct avd_dev dev={.variant=&variant,.queued=false}; + struct avd_coded_fmt_ops ops={};struct fmt_desc fmt={.ops=&ops}; + struct avd_ctx ctx={.dev=&dev,.coded_fmt_desc=&fmt}; + struct avd_buf program={};struct avd_frame f={.prog=&program,.ctx=&ctx};dev.building=&f; + struct avd_decoded_buffer dst={};struct vb2_v4l2_buffer src={};u8 coded[512]={}; + dst.base.vb.vb2_buf.dma=0x230000000ULL;src.vb2_buf=(struct vb2_buffer){.dma=0x220000000ULL,.cpu=coded,.payload=512}; + current_dst=&dst.base.vb;current_src=&src;ctx.inst.addr=0x205000000ULL; + ctx.decoded_fmt.fmt.pix_mp.width=1920;ctx.decoded_fmt.fmt.pix_mp.height=1088; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; + assert(!avd_vp9_start(&ctx));struct avd_vp9_ctx *p=ctx.priv; + params=(struct v4l2_ctrl_vp9_frame){.frame_width_minus_1=1919,.frame_height_minus_1=1087,.bit_depth=8}; + updates.tx_mode=V4L2_VP9_TX_MODE_SELECT; + const u32 flags[]={V4L2_VP9_FRAME_FLAG_KEY_FRAME|V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX, + V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX,V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE|V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX, + 0,V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT|V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE, + V4L2_VP9_FRAME_FLAG_KEY_FRAME|V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX|V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE}; + for(unsigned int i=0;ibufs.counts.cpu; + for(size_t j=0;j<528;j++){c->eob_0[j]=16;c->ref_cnt[j*4]=1;c->ref_cnt[j*4+1]=4;c->ref_cnt[j*4+2]=11;c->ref_cnt[j*4+3]=8;} + avd_vp9_done(&ctx,&src,&dst.base.vb,VB2_BUF_STATE_DONE); + snprintf(path,sizeof(path),"%s/context-%u.bin",argv[1],i);out=fopen(path,"wb");assert(out); + assert(fwrite(p->frame_context,1,sizeof(p->frame_context),out)==sizeof(p->frame_context));fclose(out); + } + avd_vp9_stop(&ctx);free(program.cpu); +} +''' + for name, body in (('before',old),('after',vp9)): + target = OUT / ('legacy-' + name) + target.mkdir(exist_ok=True) + src = pre + without_includes(helper_h) + without_includes(helper_c) + src += without_includes((AVD/'avd-inst.h').read_text()) + src += without_includes(body[body.index('#define VP9_Q_IDX'):body.index('static enum avd_image_fmt avd_vp9_get_image_fmt')]) + src += function((AVD/'avd-backend-open.c').read_text(),'avd_open_build_program') + src += '\nstatic int avd_submit_job(struct avd_ctx *ctx) { return avd_open_build_program(ctx); }\n' + src += '#define WORK(p) ' + ('((p)->b)' if name=='after' else '(p)') + '\n' + legacy_main + src = src.replace('struct { u32 bytesperline; } plane_fmt[1];','u32 width,height; struct { u32 bytesperline; } plane_fmt[1];') + (target/'legacy.c').write_text(src) + subprocess.run(cmd[:-3]+[str(target/'legacy.c'),'-o',str(target/'legacy')],check=True) + subprocess.run([str(target/'legacy'),str(target)],check=True) + for p in sorted((OUT/'legacy-before').glob('*.bin')): + assert p.read_bytes()==(OUT/'legacy-after'/p.name).read_bytes(),p.name + print('PASS: legacy VP9 programs and adapted frame contexts byte-identical to '+base_ref) diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem.c index 311e2e08d066d..dd47ccd7f7d71 100644 --- a/drivers/media/v4l2-core/v4l2-mem2mem.c +++ b/drivers/media/v4l2-core/v4l2-mem2mem.c @@ -503,6 +503,34 @@ void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev, } EXPORT_SYMBOL(v4l2_m2m_job_finish); +void v4l2_m2m_job_finish_detach(struct v4l2_m2m_dev *m2m_dev, + struct v4l2_m2m_ctx *m2m_ctx) +{ + struct vb2_v4l2_buffer *src_buf, *dst_buf; + bool schedule_next = false; + unsigned long flags; + + spin_lock_irqsave(&m2m_dev->job_spinlock, flags); + if (WARN_ON(m2m_dev->curr_ctx != m2m_ctx)) + goto unlock; + src_buf = v4l2_m2m_next_src_buf(m2m_ctx); + dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx); + /* Partial slices must use buf_done_and_job_finish() and keep CAPTURE. */ + if (WARN_ON(!src_buf || !dst_buf || + (src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF))) + goto unlock; + + v4l2_m2m_src_buf_remove(m2m_ctx); + v4l2_m2m_dst_buf_remove(m2m_ctx); + dst_buf->is_held = false; + schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx); + unlock: + spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags); + if (schedule_next) + v4l2_m2m_schedule_next_job(m2m_dev, m2m_ctx); +} +EXPORT_SYMBOL_GPL(v4l2_m2m_job_finish_detach); + void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev, struct v4l2_m2m_ctx *m2m_ctx, enum vb2_buffer_state state) diff --git a/include/media/v4l2-mem2mem.h b/include/media/v4l2-mem2mem.h index 31de25d792b98..5284bc506fc59 100644 --- a/include/media/v4l2-mem2mem.h +++ b/include/media/v4l2-mem2mem.h @@ -200,6 +200,24 @@ void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *m2m_ctx); void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev, struct v4l2_m2m_ctx *m2m_ctx); +/** + * v4l2_m2m_job_finish_detach() - detach buffers and release a dispatch + * @m2m_dev: mem2mem device + * @m2m_ctx: context of the running job + * + * Remove the current source and destination from their ready lists and finish + * the running job, without completing either buffer. The driver must save both + * buffer pointers before calling, retain ownership until hardware has stopped + * accessing them, and eventually complete them with v4l2_m2m_buf_done(). + * + * This may be used by pipelined drivers on HOLD_CAPTURE-capable queues only + * for a source without V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF. Partial slices must + * instead use v4l2_m2m_buf_done_and_job_finish() to retain the capture buffer. + * The destination is no longer held or visible to subsequent m2m scheduling. + */ +void v4l2_m2m_job_finish_detach(struct v4l2_m2m_dev *m2m_dev, + struct v4l2_m2m_ctx *m2m_ctx); + /** * v4l2_m2m_buf_done_and_job_finish() - return source/destination buffers with * state and inform the framework that a job has been finished and have it diff --git a/tools/asahi/avd-codec-tail-tests.py b/tools/asahi/avd-codec-tail-tests.py index 31616efc2651c..a7f2e039957f1 100755 --- a/tools/asahi/avd-codec-tail-tests.py +++ b/tools/asahi/avd-codec-tail-tests.py @@ -17,7 +17,7 @@ import tempfile ROOT = Path(__file__).resolve().parents[2] -SOURCE = "drivers/media/platform/apple/avd/avd-drv.c" +SOURCE = "drivers/media/platform/apple/avd/avd-backend-open.c" HEADER = "drivers/media/platform/apple/avd/avd.h" @@ -38,13 +38,14 @@ def build_source(body, defines): #define dev_info(...) ((void)0) ''' + defines + r''' struct firmware { const unsigned char *data; size_t size; }; -struct variant { unsigned int capabilities; }; +struct variant { unsigned int capabilities, fw_caps_required; }; struct avd_dev { struct firmware *fw; const struct variant *variant; - unsigned int capabilities; + unsigned int capabilities, fw_capabilities; }; -static char *strnstr(const char *s, const char *needle, size_t n) +#define strnstr fixture_strnstr +static char *fixture_strnstr(const char *s, const char *needle, size_t n) { size_t len = strlen(needle); for (size_t i = 0; i + len <= n && s[i]; i++) @@ -52,10 +53,10 @@ def build_source(body, defines): return (char *)s + i; return NULL; } +''' + body + r''' static unsigned int probe_tail(struct avd_dev *avd) { - const char *codec_info = NULL; -''' + body + r''' + avd_open_capabilities(avd); return avd->capabilities; } int main(int argc, char **argv) @@ -67,12 +68,14 @@ def build_source(body, defines): if (!data) return 2; struct firmware fw = {data, size}; - struct variant variant = {strtoul(argv[3], NULL, 0)}; - struct avd_dev avd = {&fw, &variant, 0}; + struct variant variant = {strtoul(argv[3], NULL, 0), 0}; + struct avd_dev avd = {&fw, &variant, 0, 0}; size_t len = strlen(argv[2]); if (size >= AVD_CODEC_INFO_MAX_SIZE && len <= AVD_CODEC_INFO_MAX_SIZE) memcpy(data + size - AVD_CODEC_INFO_MAX_SIZE, argv[2], len); unsigned int actual = probe_tail(&avd); + if (actual != avd.fw_capabilities) + return 1; unsigned int expected = strtoul(argv[4], NULL, 0); printf("size=%zu variant=%u capabilities=%u expected=%u\n", size, variant.capabilities, actual, expected); @@ -122,12 +125,12 @@ def main(): args = parser.parse_args() args.output.mkdir(parents=True, exist_ok=False) source, header = read(SOURCE, args.ref), read(HEADER, args.ref) - start = source.index("\t/* Determine firmware and HW variant capabilities */") - end = source.index("\n\tret = dma_set_mask_and_coherent", start) + start = source.index("static void avd_open_capabilities(") + end = source.index("static int avd_open_request_firmware(", start) body = source[start:end] - wanted = ["AVD_CODEC_INFO_MAX_SIZE", "AVD_CAPABILITY_AV1", "AVD_CAPABILITY_H264", + wanted = ["AVD_CAPABILITY_AV1", "AVD_CAPABILITY_H264", "AVD_CAPABILITY_HEVC", "AVD_CAPABILITY_VP9"] - defines = "" + defines = "#define AVD_CODEC_INFO_MAX_SIZE 0x40\n#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))\n#define u32 unsigned int\n" for macro in wanted: matches = re.findall(r"^#define\s+" + macro + r"\s+[^\n]+$", source + "\n" + header, re.M) @@ -148,16 +151,15 @@ def main(): (128, "legacy", 5, 5)] variants = [("control-before", body)] if args.mutations: - anchor = "\tif (avd->fw->size >= AVD_CODEC_INFO_MAX_SIZE)\n" + anchor = "!avd->fw || avd->fw->size < 0x40" if body.count(anchor) != 1: raise ValueError("size-guard mutation did not match exactly once") - variants += [("no-size-guard", body.replace(anchor, "")), + variants += [("no-size-guard", body.replace(anchor, "!avd->fw")), ("no-variant-intersection", body.replace( - "\t\tunsigned int variant_caps = avd->variant->capabilities;\n", "" - ).replace("caps |= variant_caps & ", "caps |= ")), + "avd->variant->capabilities & codecs[i].bit", "codecs[i].bit")), ("no-legacy-fallback", body.replace( - "avd->capabilities = avd->variant->capabilities;", - "avd->capabilities = 0;"))] + "avd->variant->capabilities &\n\t\t\t ~avd->variant->fw_caps_required", + "0"))] variants.append(("control-after", body)) results = [run_variant(args.output, name, text, defines, cases) for name, text in variants]